Skip to content
This repository was archived by the owner on Sep 2, 2021. It is now read-only.

Commit 14a0758

Browse files
saurabh95nethip
saurabh95
authored and
nethip
committed
Correct UTF8 detection which was being detected as ISO-8859-1 (#612)
* Correcte UTF8 detection which was being detected as ISO-8859-1 * File is read only once now in mac * minor change * Added back functions which are used in windows * minor change * minor change * minor change * Added some comments related to BOM * Added some comments related to BOM
1 parent d1b1c3e commit 14a0758

4 files changed

Lines changed: 82 additions & 33 deletions

File tree

‎appshell/appshell_extensions_mac.mm‎

Lines changed: 50 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -654,46 +654,62 @@ int32 ReadFile(ExtensionString filename, ExtensionString& encoding, std::string&
654654
if (encoding == "utf8") {
655655
encoding = "UTF-8";
656656
}
657+
NSString* path = [NSString stringWithUTF8String:filename.c_str()];
658+
659+
NSStringEncoding enc;
657660
int32 error = NO_ERROR;
658-
659-
try {
660-
std::ifstream file(filename.c_str());
661-
std::stringstream ss;
662-
ss << file.rdbuf();
663-
contents = ss.str();
664-
std::string detectedCharSet;
661+
662+
NSString* fileContents = nil;
663+
if (encoding == "UTF-8") {
664+
enc = NSUTF8StringEncoding;
665+
NSError* NSerror = nil;
666+
fileContents = [NSString stringWithContentsOfFile:path encoding:enc error:&NSerror];
667+
}
668+
669+
if (fileContents)
670+
{
671+
contents = [fileContents UTF8String];
672+
// We check if the file contains BOM or not
673+
// if yes, then we set preserveBOM to true
674+
// Please note we try to read first 3 characters
675+
// again to check for BOM
676+
CheckForUTF8BOM(filename, preserveBOM);
677+
return NO_ERROR;
678+
} else {
665679
try {
666-
if (encoding == "UTF-8") {
667-
CharSetDetect ICUDetector;
668-
ICUDetector(contents.c_str(), contents.size(), detectedCharSet);
669-
}
670-
else {
671-
detectedCharSet = encoding;
672-
}
673-
if (detectedCharSet == "UTF-16LE" || detectedCharSet == "UTF-16BE") {
674-
return ERR_UNSUPPORTED_UTF16_ENCODING;
675-
}
676-
if (detectedCharSet != "UTF-8") {
677-
try {
680+
std::ifstream file(filename.c_str());
681+
std::stringstream ss;
682+
ss << file.rdbuf();
683+
contents = ss.str();
684+
std::string detectedCharSet;
685+
try {
686+
if (encoding == "UTF-8") {
687+
CharSetDetect ICUDetector;
688+
ICUDetector(contents.c_str(), contents.size(), detectedCharSet);
689+
}
690+
else {
691+
detectedCharSet = encoding;
692+
}
693+
if (detectedCharSet == "UTF-16LE" || detectedCharSet == "UTF-16BE") {
694+
return ERR_UNSUPPORTED_UTF16_ENCODING;
695+
}
696+
if (!detectedCharSet.empty()) {
678697
std::transform(detectedCharSet.begin(), detectedCharSet.end(), detectedCharSet.begin(), ::toupper);
679698
DecodeContents(contents, detectedCharSet);
680699
encoding = detectedCharSet;
681-
} catch (...) {
682-
error = ERR_DECODE_FILE_FAILED;
683700
}
684-
}
685-
else {
686-
CheckAndRemoveUTF8BOM(contents, preserveBOM);
701+
else {
702+
error = ERR_UNSUPPORTED_ENCODING;
703+
}
704+
} catch (...) {
705+
error = ERR_UNSUPPORTED_ENCODING;
687706
}
688707
} catch (...) {
689-
error = ERR_UNSUPPORTED_ENCODING;
708+
error = ERR_CANT_READ;
690709
}
691-
} catch (...) {
692-
error = ERR_CANT_READ;
693710
}
694711

695-
return error;
696-
}
712+
return error;}
697713

698714
int32 WriteFile(ExtensionString filename, std::string contents, ExtensionString encoding, bool preserveBOM)
699715
{
@@ -711,13 +727,18 @@ int32 WriteFile(ExtensionString filename, std::string contents, ExtensionString
711727
error = ERR_ENCODE_FILE_FAILED;
712728
}
713729
} else if (encoding == "UTF-8" && preserveBOM) {
730+
// The file originally contained BOM chars
731+
// so we prepend BOM chars
714732
contents = UTF8_BOM + contents;
715733
}
716734

717735
try {
718736
std::ofstream file;
719737
file.open (filenameStr);
720738
file << contents;
739+
if (file.fail()) {
740+
error = ERR_CANT_WRITE;
741+
}
721742
file.close();
722743
} catch (...) {
723744
return ERR_CANT_WRITE;

‎appshell/appshell_extensions_platform.cpp‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "appshell/appshell_extensions_platform.h"
22
#include <unicode/ucsdet.h>
33
#include <unicode/ucnv.h>
4+
#include <fstream>
45

56
#define UTF8_BOM "\xEF\xBB\xBF"
67

@@ -92,9 +93,28 @@ void DecodeContents(std::string &contents, const std::string& encoding) {
9293
#endif
9394

9495
void CheckAndRemoveUTF8BOM(std::string& contents, bool& preserveBOM) {
95-
if (contents.length() >= 3 && contents.substr(0,3) == UTF8_BOM) {
96-
contents.erase(0,3);
96+
if (contents.length() >= 3 && contents.substr(0, 3) == UTF8_BOM) {
9797
preserveBOM = true;
98+
contents.erase(0, 3);
9899
}
99100
}
100101

102+
void CheckForUTF8BOM(const std::string& filename, bool& preserveBOM) {
103+
try {
104+
std::ifstream file(filename.c_str());
105+
int ch1, ch2, ch3;
106+
ch1 = ch2 = ch3 = 0;
107+
if (file.good())
108+
ch1 = file.get();
109+
if (file.good())
110+
ch2 = file.get();
111+
if (file.good())
112+
ch3 = file.get();
113+
if (ch1 == 0xef && ch2 == 0xbb && ch3 == 0xbf) {
114+
preserveBOM = true;
115+
}
116+
}
117+
catch (...) {
118+
}
119+
}
120+

‎appshell/appshell_extensions_platform.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ void DecodeContents(std::string &contents, const std::string& encoding);
117117

118118
void CheckAndRemoveUTF8BOM(std::string& contents, bool& preserveBOM);
119119

120+
void CheckForUTF8BOM(const std::string& filename, bool& preserveBOM);
121+
120122
// Native extension code. These are implemented in appshell_extensions_mac.mm
121123
// and appshell_extensions_win.cpp
122124
int32 OpenLiveBrowser(ExtensionString argURL, bool enableRemoteDebugging);

‎appshell/appshell_extensions_win.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -985,7 +985,7 @@ int32 ReadFile(ExtensionString filename, ExtensionString& encoding, std::string&
985985
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> conv;
986986
detectedCharSet = conv.to_bytes(encoding);
987987
}
988-
if (detectedCharSet == "UTF-16LE" || detectedCharSet == "UTF-16LE") {
988+
if (detectedCharSet == "UTF-16LE" || detectedCharSet == "UTF-16BE") {
989989
return ERR_UNSUPPORTED_UTF16_ENCODING;
990990
}
991991
std::transform(detectedCharSet.begin(), detectedCharSet.end(), detectedCharSet.begin(), ::toupper);
@@ -1012,7 +1012,10 @@ int32 ReadFile(ExtensionString filename, ExtensionString& encoding, std::string&
10121012
}
10131013
}
10141014
if (encoding == L"UTF-8") {
1015-
CheckAndRemoveUTF8BOM(contents, preserveBOM);
1015+
// If file starts with BOM chars, then
1016+
// we set preserveBOM to true, so that
1017+
// while writing we can preprend the BOM
1018+
CheckAndRemoveUTF8BOM(contents, preserveBOM);
10161019
}
10171020
}
10181021
else {
@@ -1080,6 +1083,9 @@ int32 WriteFile(ExtensionString filename, std::string contents, ExtensionString
10801083
error = ERR_ENCODE_FILE_FAILED;
10811084
}
10821085
}
1086+
// We check if the file originally contained BOM chars
1087+
// if yes, then we prepend BOM chars to file
1088+
// Currently BOM is supported only for UTF-8
10831089
if (encoding == L"UTF-8" && preserveBOM) {
10841090
contents = UTF8_BOM + contents;
10851091
}

0 commit comments

Comments
 (0)