View Issue Details

IDProjectCategoryView StatusLast Update
0019742LazarusRTLpublic2011-10-03 17:09
ReporterMarkus MüllerAssigned ToFelipe Monteiro de Carvalho 
PrioritynormalSeverityfeatureReproducibilityN/A
Status resolvedResolutionfixed 
PlatformWindowsOSWindowsOS VersionAll
Product VersionProduct Build 
Target VersionFixed in Version0.9.31 (SVN) 
Summary0019742: New TFileStreamUTF8 and TStrings.LoadFromFileUTF8 / SaveToFileUTF8
DescriptionI have with Windows the problem, I cannot save/read TStringList when there are UTF8 Characters like łążćóß in the pathname.

So I have include the UTF8 feature into the TStrings.

I have insert the changes after this installation: "Lazarus-0.9.31-31683-fpc-2.5.1-20110713-win32.exe"
Additional InformationI cannot test the new feature, then I do not know how I can rebuld the RTL code :-(
Tagsrtl
Fixed in Revision32656
LazTarget-
Widgetset
Attached Files
  • fpc.diff (4,682 bytes)
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc
    617d616
    <     procedure LoadFromFileUTF8(const FileName: string); virtual;
    621d619
    <     procedure SaveToFileUTF8(const FileName: string); virtual;
    871,883c869
    <   { TFileStreamUTF8 class }
    < 
    <     TFileStreamUTF8 = class(THandleStream)
    <     Private
    <       FFileName : String;
    <     public
    <       constructor Create(const AFileName: string; Mode: Word);
    <       constructor Create(const AFileName: string; Mode: Word; Rights: Cardinal);
    <       destructor Destroy; override;
    <       property FileName : String Read FFilename;
    <     end;
    < 
    <   { TCustomMemoryStream abstract class }
    ---
    > { TCustomMemoryStream abstract class }
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\streams.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\streams.inc
    516,559d515
    < {*                             TFileStreamUTF8                              *}
    < {****************************************************************************}
    < 
    < constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word);
    < 
    < begin
    <   FFileName:=AFileName;
    <   If (Mode and fmCreate) > 0 then
    <     FHandle:=FileCreateUTF8(AFileName,Mode,438)
    <   else
    <     FHAndle:=FileOpenUTF8(AFileName,Mode);
    < 
    <   If (THandle(FHandle)=feInvalidHandle) then
    <     If Mode=fmcreate then
    <       raise EFCreateError.createfmt(SFCreateError,[AFileName])
    <     else
    <       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
    < end;
    < 
    < 
    < constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word; Rights: Cardinal);
    < 
    < begin
    <   FFileName:=AFileName;
    <   If (Mode and fmCreate) > 0 then
    <     FHandle:=FileCreateUTF8(AFileName,Mode,438)
    <   else
    <     FHAndle:=FileOpenUTF8(AFileName,Mode);
    < 
    <   If (THandle(FHandle)=feInvalidHandle) then
    <     If Mode=fmcreate then
    <       raise EFCreateError.createfmt(SFCreateError,[AFileName])
    <     else
    <       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
    < end;
    < 
    < 
    < destructor TFileStreamUTF8.Destroy;
    < 
    < begin
    <   FileClose(FHandle);
    < end;
    < 
    < {****************************************************************************}
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc
    778,788d777
    < Procedure TStrings.LoadFromFileUTF8(const FileName: string);
    < Var
    <         TheStream : TFileStreamUTF8;
    < begin
    <   TheStream:=TFileStreamUTF8.Create(FileName,fmOpenRead or fmShareDenyWrite);
    <   try
    <     LoadFromStream(TheStream);
    <   finally
    <     TheStream.Free;
    <   end;
    < end;
    858,869d846
    < Procedure TStrings.SaveToFileUTF8(const FileName: string);
    < 
    < Var TheStream : TFileStreamUTF8;
    < 
    < begin
    <   TheStream:=TFileStreamUTF8.Create(FileName,fmCreate);
    <   try
    <     SaveToStream(TheStream);
    <   finally
    <     TheStream.Free;
    <   end;
    < end;
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc
    79d78
    < Function FileOpenUT8 (Const FileName : string; Mode : Integer) : THandle;
    83,85d81
    < Function FileCreateUTF8 (Const FileName : String) : THandle;
    < Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
    < Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\win\sysutils.pp Lazarus_Orig\fpc\2.5.1\source\rtl\win\sysutils.pp
    236,243d235
    < Function FileOpenUTF8 (Const FileName : string; Mode : Integer) : THandle;
    < begin
    <   result := CreateFileW(PWideChar(UTF8Decode(FileName)), dword(AccessMode[Mode and 3]),
    <                        dword(ShareModes[(Mode and $F0) shr 4]), nil, OPEN_EXISTING,
    <                        FILE_ATTRIBUTE_NORMAL, 0);
    <   //if fail api return feInvalidHandle (INVALIDE_HANDLE=feInvalidHandle=-1)
    < end;
    < 
    257,272d248
    <                        dword(ShareModes[(ShareMode and $F0) shr 4]), nil, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
    < end;
    < 
    < Function FileCreateUTF8 (Const FileName : String) : THandle;
    < begin
    <   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, 0);
    < end;
    < 
    < Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
    < begin
    <   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, Rights);
    < end;
    < 
    < Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
    < begin
    <   Result := CreateFileW(PWideChar(UTF8Decode(FileName)), GENERIC_READ or GENERIC_WRITE,
    
    fpc.diff (4,682 bytes)
  • fpc_2.diff (4,105 bytes)
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc
    617d616
    <     procedure LoadFromFileUTF8(const FileName: string); virtual;
    621d619
    <     procedure SaveToFileUTF8(const FileName: string); virtual;
    867,868d864
    <     constructor CreateUTF8(const AFileName: string; Mode: Word);
    <     constructor CreateUTF8(const AFileName: string; Mode: Word; Rights: Cardinal);
    873c869
    <   { TCustomMemoryStream abstract class }
    ---
    > { TCustomMemoryStream abstract class }
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\streams.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\streams.inc
    509,542d508
    < constructor TFileStream.CreateUTF8(const AFileName: string; Mode: Word);
    < 
    < begin
    <   FFileName:=AFileName;
    <   If (Mode and fmCreate) > 0 then
    <     FHandle:=FileCreateUTF8(AFileName,Mode,438)
    <   else
    <     FHAndle:=FileOpenUTF8(AFileName,Mode);
    < 
    <   If (THandle(FHandle)=feInvalidHandle) then
    <     If Mode=fmcreate then
    <       raise EFCreateError.createfmt(SFCreateError,[AFileName])
    <     else
    <       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
    < end;
    < 
    < 
    < constructor TFileStream.CreateUTF8(const AFileName: string; Mode: Word; Rights: Cardinal);
    < 
    < begin
    <   FFileName:=AFileName;
    <   If (Mode and fmCreate) > 0 then
    <     FHandle:=FileCreateUTF8(AFileName,Mode,Rights)
    <   else
    <     FHAndle:=FileOpenUTF8(AFileName,Mode);
    < 
    <   If (THandle(FHandle)=feInvalidHandle) then
    <     If Mode=fmcreate then
    <       raise EFCreateError.createfmt(SFCreateError,[AFileName])
    <     else
    <       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
    < end;
    < 
    < 
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc
    778,788d777
    < Procedure TStrings.LoadFromFileUTF8(const FileName: string);
    < Var
    <         TheStream : TFileStream;
    < begin
    <   TheStream:=TFileStream.CreateUTF8(FileName,fmOpenRead or fmShareDenyWrite);
    <   try
    <     LoadFromStream(TheStream);
    <   finally
    <     TheStream.Free;
    <   end;
    < end;
    858,869d846
    < Procedure TStrings.SaveToFileUTF8(const FileName: string);
    < 
    < Var TheStream : TFileStream;
    < 
    < begin
    <   TheStream:=TFileStream.CreateUTF8(FileName,fmCreate);
    <   try
    <     SaveToStream(TheStream);
    <   finally
    <     TheStream.Free;
    <   end;
    < end;
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc
    79d78
    < Function FileOpenUT8 (Const FileName : string; Mode : Integer) : THandle;
    83,85d81
    < Function FileCreateUTF8 (Const FileName : String) : THandle;
    < Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
    < Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
    diff -r Lazarus_New\fpc\2.5.1\source\rtl\win\sysutils.pp Lazarus_Orig\fpc\2.5.1\source\rtl\win\sysutils.pp
    236,243d235
    < Function FileOpenUTF8 (Const FileName : string; Mode : Integer) : THandle;
    < begin
    <   result := CreateFileW(PWideChar(UTF8Decode(FileName)), dword(AccessMode[Mode and 3]),
    <                        dword(ShareModes[(Mode and $F0) shr 4]), nil, OPEN_EXISTING,
    <                        FILE_ATTRIBUTE_NORMAL, 0);
    <   //if fail api return feInvalidHandle (INVALIDE_HANDLE=feInvalidHandle=-1)
    < end;
    < 
    257,272d248
    <                        dword(ShareModes[(ShareMode and $F0) shr 4]), nil, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
    < end;
    < 
    < Function FileCreateUTF8 (Const FileName : String) : THandle;
    < begin
    <   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, 0);
    < end;
    < 
    < Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
    < begin
    <   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, Rights);
    < end;
    < 
    < Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
    < begin
    <   Result := CreateFileW(PWideChar(UTF8Decode(FileName)), GENERIC_READ or GENERIC_WRITE,
    
    fpc_2.diff (4,105 bytes)
  • file_stream_utf8.patch (2,660 bytes)
    Index: lcl/fileutil.pas
    ===================================================================
    --- lcl/fileutil.pas	(revision 32307)
    +++ lcl/fileutil.pas	(working copy)
    @@ -223,6 +223,18 @@
     // other
     function SysErrorMessageUTF8(ErrorCode: Integer): String;
     
    +{ TFileStreamUTF8 class }
    +type
    +  TFileStreamUTF8 = class(THandleStream)
    +  private
    +    FFileName : string;
    +  public
    +    constructor Create(const AFileName: string; Mode: Word);
    +    constructor Create(const AFileName: string; Mode: Word; Rights: Cardinal);
    +    destructor Destroy; override;
    +    property FileName : string Read FFilename;
    +  end;
    +
     implementation
     
     uses
    Index: lcl/include/winfileutil.inc
    ===================================================================
    --- lcl/include/winfileutil.inc	(revision 32307)
    +++ lcl/include/winfileutil.inc	(working copy)
    @@ -608,3 +608,61 @@
       end;
       {$endif}
     end;
    +
    +function FileOpenUTF8(Const FileName : string; Mode : Integer) : THandle;
    +const
    +  AccessMode: array[0..2] of Cardinal  = (
    +    GENERIC_READ,
    +    GENERIC_WRITE,
    +    GENERIC_READ or GENERIC_WRITE);
    +  ShareMode: array[0..4] of Integer = (
    +               0,
    +               0,
    +               FILE_SHARE_READ,
    +               FILE_SHARE_WRITE,
    +               FILE_SHARE_READ or FILE_SHARE_WRITE);
    +begin
    +  Result := CreateFileW(PWideChar(UTF8Decode(FileName)), dword(AccessMode[Mode and 3]),
    +                       dword(ShareMode[(Mode and $F0) shr 4]), nil, OPEN_EXISTING,
    +                       FILE_ATTRIBUTE_NORMAL, 0);
    +  //if fail api return feInvalidHandle (INVALIDE_HANDLE=feInvalidHandle=-1)
    +end;
    +
    +function FileCreateUTF8(Const FileName : String) : THandle;
    +begin
    +  Result := CreateFileW(PWideChar(UTF8Decode(FileName)), GENERIC_READ or GENERIC_WRITE,
    +                       0, nil, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
    +end;
    +
    +constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word);
    +var
    +  h: THandle;
    +begin
    +  FFileName:= AFileName;
    +  if Mode = fmcreate then
    +    h:= FileCreateUTF8(AFileName)
    +  else
    +    h:= FileOpenUTF8(AFileName, Mode);
    +
    +  if h = feInvalidHandle then
    +  begin
    +    if Mode = fmCreate then
    +      raise EFCreateError.createfmt({SFCreateError}'Unable to create file "%s"', [AFileName])
    +    else
    +      raise EFOpenError.Createfmt({SFOpenError}'Unable to open file "%s"', [AFilename]);
    +  end else begin
    +    inherited Create(h);
    +  end;
    +end;
    +
    +constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word; Rights: Cardinal);
    +begin
    +  // Rights No support on Windows
    +  Create(AFileName, Mode);
    +end;
    +
    +destructor TFileStreamUTF8.Destroy;
    +begin
    +  FileClose(Handle);
    +end;
    +
    
    file_stream_utf8.patch (2,660 bytes)
  • stringlist_utf8.txt (988 bytes)
    TStringListUTF8 example
    
    type
      TStringListUTF8 = class(TStringList)
      protected
        function DoCompareText(const s1,s2 : string) : PtrInt; override;
        procedure LoadFromFile(const FileName: string); override;
        procedure SaveToFile(const FileName: string); override;
      public
      end;
    
    function TStringListUTF8.DoCompareText(const s1, s2: string): PtrInt;
    begin
      // Do not use AnsiCompareText for UTF8!!!
      Result:= CompareText(s1, s2);
    end;
    
    procedure TStringListUTF8.LoadFromFile(const FileName: string);
    var
      TheStream: TFileStreamUTF8;
    begin
      TheStream:= TFileStreamUTF8.Create(FileName, fmOpenRead or fmShareDenyWrite);
      try
        LoadFromStream(TheStream);
      finally
        TheStream.Free;
      end;
    end;
    
    procedure TStringListUTF8.SaveToFile(const FileName: string);
    var
      TheStream: TFileStreamUTF8;
    begin
      TheStream:=TFileStreamUTF8.Create(FileName,fmCreate);
      try
        SaveToStream(TheStream);
      finally
        TheStream.Free;
      end;
    end;
    
    
    stringlist_utf8.txt (988 bytes)

Relationships

related to 0019743 resolvedFelipe Monteiro de Carvalho New CopyFileUTF8() 

Activities

2011-07-14 23:07

 

fpc.diff (4,682 bytes)
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc
617d616
<     procedure LoadFromFileUTF8(const FileName: string); virtual;
621d619
<     procedure SaveToFileUTF8(const FileName: string); virtual;
871,883c869
<   { TFileStreamUTF8 class }
< 
<     TFileStreamUTF8 = class(THandleStream)
<     Private
<       FFileName : String;
<     public
<       constructor Create(const AFileName: string; Mode: Word);
<       constructor Create(const AFileName: string; Mode: Word; Rights: Cardinal);
<       destructor Destroy; override;
<       property FileName : String Read FFilename;
<     end;
< 
<   { TCustomMemoryStream abstract class }
---
> { TCustomMemoryStream abstract class }
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\streams.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\streams.inc
516,559d515
< {*                             TFileStreamUTF8                              *}
< {****************************************************************************}
< 
< constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word);
< 
< begin
<   FFileName:=AFileName;
<   If (Mode and fmCreate) > 0 then
<     FHandle:=FileCreateUTF8(AFileName,Mode,438)
<   else
<     FHAndle:=FileOpenUTF8(AFileName,Mode);
< 
<   If (THandle(FHandle)=feInvalidHandle) then
<     If Mode=fmcreate then
<       raise EFCreateError.createfmt(SFCreateError,[AFileName])
<     else
<       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
< end;
< 
< 
< constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word; Rights: Cardinal);
< 
< begin
<   FFileName:=AFileName;
<   If (Mode and fmCreate) > 0 then
<     FHandle:=FileCreateUTF8(AFileName,Mode,438)
<   else
<     FHAndle:=FileOpenUTF8(AFileName,Mode);
< 
<   If (THandle(FHandle)=feInvalidHandle) then
<     If Mode=fmcreate then
<       raise EFCreateError.createfmt(SFCreateError,[AFileName])
<     else
<       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
< end;
< 
< 
< destructor TFileStreamUTF8.Destroy;
< 
< begin
<   FileClose(FHandle);
< end;
< 
< {****************************************************************************}
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc
778,788d777
< Procedure TStrings.LoadFromFileUTF8(const FileName: string);
< Var
<         TheStream : TFileStreamUTF8;
< begin
<   TheStream:=TFileStreamUTF8.Create(FileName,fmOpenRead or fmShareDenyWrite);
<   try
<     LoadFromStream(TheStream);
<   finally
<     TheStream.Free;
<   end;
< end;
858,869d846
< Procedure TStrings.SaveToFileUTF8(const FileName: string);
< 
< Var TheStream : TFileStreamUTF8;
< 
< begin
<   TheStream:=TFileStreamUTF8.Create(FileName,fmCreate);
<   try
<     SaveToStream(TheStream);
<   finally
<     TheStream.Free;
<   end;
< end;
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc
79d78
< Function FileOpenUT8 (Const FileName : string; Mode : Integer) : THandle;
83,85d81
< Function FileCreateUTF8 (Const FileName : String) : THandle;
< Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
< Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
diff -r Lazarus_New\fpc\2.5.1\source\rtl\win\sysutils.pp Lazarus_Orig\fpc\2.5.1\source\rtl\win\sysutils.pp
236,243d235
< Function FileOpenUTF8 (Const FileName : string; Mode : Integer) : THandle;
< begin
<   result := CreateFileW(PWideChar(UTF8Decode(FileName)), dword(AccessMode[Mode and 3]),
<                        dword(ShareModes[(Mode and $F0) shr 4]), nil, OPEN_EXISTING,
<                        FILE_ATTRIBUTE_NORMAL, 0);
<   //if fail api return feInvalidHandle (INVALIDE_HANDLE=feInvalidHandle=-1)
< end;
< 
257,272d248
<                        dword(ShareModes[(ShareMode and $F0) shr 4]), nil, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
< end;
< 
< Function FileCreateUTF8 (Const FileName : String) : THandle;
< begin
<   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, 0);
< end;
< 
< Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
< begin
<   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, Rights);
< end;
< 
< Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
< begin
<   Result := CreateFileW(PWideChar(UTF8Decode(FileName)), GENERIC_READ or GENERIC_WRITE,
fpc.diff (4,682 bytes)

Marco van de Voort

2011-07-14 23:10

manager   ~0049993

Last edited: 2011-07-14 23:12

I don't think this should be committed. Having an -UTF8 version of everything is Lazarus feature, not FPC, and it will go away with proper language unicode support. If there is a temporary need for such stuff, it should be kept in Lazarus.

(Delphi 2009+ adds an codepage enum to all load-x and save-x with -x being file or stream for this)

Markus Müller

2011-07-15 07:45

reporter   ~0049998

Last edited: 2011-07-15 08:09

My Problem: Only in the RTL ist the TFileStrem deklared and when this cannot UFT8 characters, then you cannot update the LCL in complet code.

I have update the diff file and not use a TFileStreamUTF8. I have implement the Constructor CreateUTF8 into TFileStream. I think it is better.

This problem is only for Windows, the the Linux File System is working allways with UFT8.

2011-07-15 07:57

 

fpc_2.diff (4,105 bytes)
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\classesh.inc
617d616
<     procedure LoadFromFileUTF8(const FileName: string); virtual;
621d619
<     procedure SaveToFileUTF8(const FileName: string); virtual;
867,868d864
<     constructor CreateUTF8(const AFileName: string; Mode: Word);
<     constructor CreateUTF8(const AFileName: string; Mode: Word; Rights: Cardinal);
873c869
<   { TCustomMemoryStream abstract class }
---
> { TCustomMemoryStream abstract class }
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\streams.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\streams.inc
509,542d508
< constructor TFileStream.CreateUTF8(const AFileName: string; Mode: Word);
< 
< begin
<   FFileName:=AFileName;
<   If (Mode and fmCreate) > 0 then
<     FHandle:=FileCreateUTF8(AFileName,Mode,438)
<   else
<     FHAndle:=FileOpenUTF8(AFileName,Mode);
< 
<   If (THandle(FHandle)=feInvalidHandle) then
<     If Mode=fmcreate then
<       raise EFCreateError.createfmt(SFCreateError,[AFileName])
<     else
<       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
< end;
< 
< 
< constructor TFileStream.CreateUTF8(const AFileName: string; Mode: Word; Rights: Cardinal);
< 
< begin
<   FFileName:=AFileName;
<   If (Mode and fmCreate) > 0 then
<     FHandle:=FileCreateUTF8(AFileName,Mode,Rights)
<   else
<     FHAndle:=FileOpenUTF8(AFileName,Mode);
< 
<   If (THandle(FHandle)=feInvalidHandle) then
<     If Mode=fmcreate then
<       raise EFCreateError.createfmt(SFCreateError,[AFileName])
<     else
<       raise EFOpenError.Createfmt(SFOpenError,[AFilename]);
< end;
< 
< 
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\classes\stringl.inc
778,788d777
< Procedure TStrings.LoadFromFileUTF8(const FileName: string);
< Var
<         TheStream : TFileStream;
< begin
<   TheStream:=TFileStream.CreateUTF8(FileName,fmOpenRead or fmShareDenyWrite);
<   try
<     LoadFromStream(TheStream);
<   finally
<     TheStream.Free;
<   end;
< end;
858,869d846
< Procedure TStrings.SaveToFileUTF8(const FileName: string);
< 
< Var TheStream : TFileStream;
< 
< begin
<   TheStream:=TFileStream.CreateUTF8(FileName,fmCreate);
<   try
<     SaveToStream(TheStream);
<   finally
<     TheStream.Free;
<   end;
< end;
diff -r Lazarus_New\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc Lazarus_Orig\fpc\2.5.1\source\rtl\objpas\sysutils\filutilh.inc
79d78
< Function FileOpenUT8 (Const FileName : string; Mode : Integer) : THandle;
83,85d81
< Function FileCreateUTF8 (Const FileName : String) : THandle;
< Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
< Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
diff -r Lazarus_New\fpc\2.5.1\source\rtl\win\sysutils.pp Lazarus_Orig\fpc\2.5.1\source\rtl\win\sysutils.pp
236,243d235
< Function FileOpenUTF8 (Const FileName : string; Mode : Integer) : THandle;
< begin
<   result := CreateFileW(PWideChar(UTF8Decode(FileName)), dword(AccessMode[Mode and 3]),
<                        dword(ShareModes[(Mode and $F0) shr 4]), nil, OPEN_EXISTING,
<                        FILE_ATTRIBUTE_NORMAL, 0);
<   //if fail api return feInvalidHandle (INVALIDE_HANDLE=feInvalidHandle=-1)
< end;
< 
257,272d248
<                        dword(ShareModes[(ShareMode and $F0) shr 4]), nil, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
< end;
< 
< Function FileCreateUTF8 (Const FileName : String) : THandle;
< begin
<   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, 0);
< end;
< 
< Function FileCreateUTF8 (Const FileName : String; Rights:longint) : THandle;
< begin
<   FileCreateUTF8:=FileCreateUTF8(FileName, fmShareExclusive, Rights);
< end;
< 
< Function FileCreateUTF8 (Const FileName : String; ShareMode : Integer; Rights : Integer) : THandle;
< begin
<   Result := CreateFileW(PWideChar(UTF8Decode(FileName)), GENERIC_READ or GENERIC_WRITE,
fpc_2.diff (4,105 bytes)

Maxim Ganetsky

2011-07-15 21:35

developer   ~0050013

@Markus: do you know that there are UTF8ToSys/SysToUTF8 functions in Lazarus to solve your problem?

I agree with Marco that this shouldn't be committed.

Markus Müller

2011-07-16 06:50

reporter   ~0050015

Last edited: 2011-07-16 07:01

It work not korrectly.
My PC ist german. When I want use Files with polish language like żłąęńćźśó then this routine cannot translate this characters into the Ansi Sys Codetable.

I think so: In normal explorer, I can see this files. With TOpenDialg I can select this Files, but with TFileStream cannot I read this files, with UTF8ToSys the same result. TStringList and the LCL use this TFileStream.

With TPicture is the same Problem (TPicture.LoadFromFile) but when I have TFileStream.CreateUTF8() then I can do TPicture.LoadFromStream. For this, I think it is a smaller work when TFileStream.Create use the WideString Files, then all functions (Txxx.LoadFromFile / Txxx.SaveToFile) work with one change correctly.

Marco van de Voort

2011-07-26 16:19

manager   ~0050184

Possibly. But with full unicode support being actively worked on, I'm not going to commit non delphi compatible workarounds. That will only cause a snowball effect.

Markus Müller

2011-07-27 07:47

reporter   ~0050194

What is Delphi? I use Delphi at last a lot of years ago. Lazarus can be better as Delphi ;-). Be self a step bevore. The V2.5.x is a good step to do this change.

I think this small chage in TFileStream.Create have a big step more in the full unicode support.

Now, I have must programming a workaround, for the problem and it work now right. But all workaround is always bullshit and can make problems in the feature and other people must every time look what my code do.

sorry, that I urge so eager and my not so good English.
Thx, regards.

Marco van de Voort

2011-07-27 14:22

manager   ~0050208

That's exactly the problem. What you propose is just a workaround too, and you are asking to integrate it into code that must be longterm supported.

Markus Müller

2011-07-27 18:06

reporter   ~0050210

I think you make the right decision to solve the problem Unicode.
I just do not know the current status of lazarus.

Takeda Matsuki

2011-07-28 01:13

reporter   ~0050228

To Mr. Marco Van de Voort ;

I agree with Markus Muller.. Lazarus is support UTF8? "Might" Yes.. Since We can saw the unicode Characters.. But inter-process of FPC (with OS System) itself is always use ANSI based, right?

e.g.

My program will call another exe to do something...
For example : I call Mpress to compressing file inside Korean Folder and the name of file is in Vietnamese Language.

TProcessUTF8.CommandLine:= 'MyAppPath\MPress\mpress.exe'+' -m' +'"'+ C:\하마의-Folder\Phải có đôi lúc-win32 TC244 A2.exe+'"' <- I set it as UTF8String

Guess.. It failed.. and it give me the report like this "FileNotFound"
But If I use "Normal Character" (a,b,c, d, ....,z) everything is working properly..

The UTF8 needs is "nothing for Unix and Linux based since they are natively support it.. But How about with WIndows?

I guess, many programmer need for UTF8/Unicode Support since this time is globalization era. If our app is fully support Unicode then we can get many users from different country and then share the files (came from different files name character) although we not use the same language characters in files name in our OS, it would be nice if we can open and do the operation with that files.

Regards,
Takeda.

Vincent Snijders

2011-09-06 00:43

manager   ~0051493

Last edited: 2011-09-06 00:44

I guess in the Lazarus project, this will be resolved as won't fix. The fix must be made into the RTL. not the LCL.

I am anxiously awaiting a Unicode RTL, so things like TProcessUTF8 can be removed from the LCL.

Felipe Monteiro de Carvalho

2011-09-07 08:23

developer   ~0051542

> I guess in the Lazarus project, this will be resolved as won't fix.

I don't think it should be a won't fix, we have FileUtil where workarounds for the lack of a UTF-8 RTL are placed, at the very least they should be placed there.

@Markus Müller

Please provide a patch for Lazarus instead. We can add TFileStreamUTF8 to FileUtil and then base for example CopyFile on it. The implementation section should go to a include file fileutilclasses.inc to keep the unit clean.

TStrings cannot be modified, but we can introduce a TStringsUTF8, TStringListUTF8

> I am anxiously awaiting a Unicode RTL, so things like TProcessUTF8 can be removed from the LCL.

I don't think it will solve anything for Lazarus if the Unicode RTL does not support UTF-8. Last time I talked with Mattias he had an oppinion similar to mine.

I see no reason and motivation to migrate the huge LCL + all other components + all apps to UTF-16, so if the RTL will support only UTF-16 it will be just as problematic as the current Ansi RTL is for Lazarus.

Markus Müller

2011-09-11 16:55

reporter   ~0051725

I have loat the last snapshot "Lazarus-0.9.31-32237-fpc-2.7.1-20110909-win32.exe" but there are this new types not includet.

malcome

2011-09-13 06:55

reporter   ~0051761

Last edited: 2011-09-13 06:56

I create ​​a patch.(Only for Windows)

2011-09-13 06:55

 

file_stream_utf8.patch (2,660 bytes)
Index: lcl/fileutil.pas
===================================================================
--- lcl/fileutil.pas	(revision 32307)
+++ lcl/fileutil.pas	(working copy)
@@ -223,6 +223,18 @@
 // other
 function SysErrorMessageUTF8(ErrorCode: Integer): String;
 
+{ TFileStreamUTF8 class }
+type
+  TFileStreamUTF8 = class(THandleStream)
+  private
+    FFileName : string;
+  public
+    constructor Create(const AFileName: string; Mode: Word);
+    constructor Create(const AFileName: string; Mode: Word; Rights: Cardinal);
+    destructor Destroy; override;
+    property FileName : string Read FFilename;
+  end;
+
 implementation
 
 uses
Index: lcl/include/winfileutil.inc
===================================================================
--- lcl/include/winfileutil.inc	(revision 32307)
+++ lcl/include/winfileutil.inc	(working copy)
@@ -608,3 +608,61 @@
   end;
   {$endif}
 end;
+
+function FileOpenUTF8(Const FileName : string; Mode : Integer) : THandle;
+const
+  AccessMode: array[0..2] of Cardinal  = (
+    GENERIC_READ,
+    GENERIC_WRITE,
+    GENERIC_READ or GENERIC_WRITE);
+  ShareMode: array[0..4] of Integer = (
+               0,
+               0,
+               FILE_SHARE_READ,
+               FILE_SHARE_WRITE,
+               FILE_SHARE_READ or FILE_SHARE_WRITE);
+begin
+  Result := CreateFileW(PWideChar(UTF8Decode(FileName)), dword(AccessMode[Mode and 3]),
+                       dword(ShareMode[(Mode and $F0) shr 4]), nil, OPEN_EXISTING,
+                       FILE_ATTRIBUTE_NORMAL, 0);
+  //if fail api return feInvalidHandle (INVALIDE_HANDLE=feInvalidHandle=-1)
+end;
+
+function FileCreateUTF8(Const FileName : String) : THandle;
+begin
+  Result := CreateFileW(PWideChar(UTF8Decode(FileName)), GENERIC_READ or GENERIC_WRITE,
+                       0, nil, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
+end;
+
+constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word);
+var
+  h: THandle;
+begin
+  FFileName:= AFileName;
+  if Mode = fmcreate then
+    h:= FileCreateUTF8(AFileName)
+  else
+    h:= FileOpenUTF8(AFileName, Mode);
+
+  if h = feInvalidHandle then
+  begin
+    if Mode = fmCreate then
+      raise EFCreateError.createfmt({SFCreateError}'Unable to create file "%s"', [AFileName])
+    else
+      raise EFOpenError.Createfmt({SFOpenError}'Unable to open file "%s"', [AFilename]);
+  end else begin
+    inherited Create(h);
+  end;
+end;
+
+constructor TFileStreamUTF8.Create(const AFileName: string; Mode: Word; Rights: Cardinal);
+begin
+  // Rights No support on Windows
+  Create(AFileName, Mode);
+end;
+
+destructor TFileStreamUTF8.Destroy;
+begin
+  FileClose(Handle);
+end;
+
file_stream_utf8.patch (2,660 bytes)

2011-09-13 08:18

 

stringlist_utf8.txt (988 bytes)
TStringListUTF8 example

type
  TStringListUTF8 = class(TStringList)
  protected
    function DoCompareText(const s1,s2 : string) : PtrInt; override;
    procedure LoadFromFile(const FileName: string); override;
    procedure SaveToFile(const FileName: string); override;
  public
  end;

function TStringListUTF8.DoCompareText(const s1, s2: string): PtrInt;
begin
  // Do not use AnsiCompareText for UTF8!!!
  Result:= CompareText(s1, s2);
end;

procedure TStringListUTF8.LoadFromFile(const FileName: string);
var
  TheStream: TFileStreamUTF8;
begin
  TheStream:= TFileStreamUTF8.Create(FileName, fmOpenRead or fmShareDenyWrite);
  try
    LoadFromStream(TheStream);
  finally
    TheStream.Free;
  end;
end;

procedure TStringListUTF8.SaveToFile(const FileName: string);
var
  TheStream: TFileStreamUTF8;
begin
  TheStream:=TFileStreamUTF8.Create(FileName,fmCreate);
  try
    SaveToStream(TheStream);
  finally
    TheStream.Free;
  end;
end;

stringlist_utf8.txt (988 bytes)

Felipe Monteiro de Carvalho

2011-10-03 17:09

developer   ~0052455

I commited with many changes in rev 32656, please test and close if ok.

Issue History

Date Modified Username Field Change
2011-07-14 23:07 Markus Müller New Issue
2011-07-14 23:07 Markus Müller File Added: fpc.diff
2011-07-14 23:09 Markus Müller Tag Attached: rtl
2011-07-14 23:10 Marco van de Voort Note Added: 0049993
2011-07-14 23:12 Marco van de Voort Note Edited: 0049993
2011-07-15 00:59 Maxim Ganetsky Relationship added related to 0019743
2011-07-15 07:45 Markus Müller Note Added: 0049998
2011-07-15 07:57 Markus Müller File Added: fpc_2.diff
2011-07-15 08:00 Markus Müller Note Edited: 0049998
2011-07-15 08:09 Markus Müller Note Edited: 0049998
2011-07-15 21:35 Maxim Ganetsky Note Added: 0050013
2011-07-16 06:50 Markus Müller Note Added: 0050015
2011-07-16 07:01 Markus Müller Note Edited: 0050015
2011-07-26 16:19 Marco van de Voort Note Added: 0050184
2011-07-27 07:47 Markus Müller Note Added: 0050194
2011-07-27 14:22 Marco van de Voort Note Added: 0050208
2011-07-27 18:06 Markus Müller Note Added: 0050210
2011-07-27 18:14 Marco van de Voort Project FPC => Lazarus
2011-07-28 01:13 Takeda Matsuki Note Added: 0050228
2011-09-06 00:43 Vincent Snijders Note Added: 0051493
2011-09-06 00:44 Vincent Snijders Note Edited: 0051493
2011-09-07 08:23 Felipe Monteiro de Carvalho Note Added: 0051542
2011-09-07 08:24 Felipe Monteiro de Carvalho Status new => assigned
2011-09-07 08:24 Felipe Monteiro de Carvalho Assigned To => Felipe Monteiro de Carvalho
2011-09-07 08:28 Felipe Monteiro de Carvalho LazTarget => -
2011-09-07 08:28 Felipe Monteiro de Carvalho Status assigned => feedback
2011-09-11 16:55 Markus Müller Note Added: 0051725
2011-09-13 06:55 malcome Note Added: 0051761
2011-09-13 06:55 malcome File Added: file_stream_utf8.patch
2011-09-13 06:56 malcome Note Edited: 0051761
2011-09-13 08:18 malcome File Added: stringlist_utf8.txt
2011-10-03 17:09 Felipe Monteiro de Carvalho Fixed in Revision => 32656
2011-10-03 17:09 Felipe Monteiro de Carvalho Status feedback => resolved
2011-10-03 17:09 Felipe Monteiro de Carvalho Fixed in Version => 0.9.31 (SVN)
2011-10-03 17:09 Felipe Monteiro de Carvalho Resolution open => fixed
2011-10-03 17:09 Felipe Monteiro de Carvalho Note Added: 0052455