View Issue Details

IDProjectCategoryView StatusLast Update
0026553LazarusLCLpublic2016-08-15 02:11
ReporterDenis KozlovAssigned ToMaxim Ganetsky 
PrioritynormalSeverityminorReproducibilityalways
Status resolvedResolutionfixed 
Product Version1.2.4Product BuildLazarus 1.2.4 r45510 FPC 2.6.4 
Target VersionFixed in Version1.8 
Summary0026553: New lines in forms break LRT and PO localization files
DescriptionIf we add new line characters (0000010 or 0000013 or 0000013#10) to TLabel.Caption (by editing *.LFM file), the resulting *.LRT and *.PO files are broken.

In such case, *.LRT file will contain literal new line characters (not escaped like in *.RST), which seem to be causing cascading problems in *.PO files:

1) Parts of multi-line captions appear as new text elements in the PO file, without context/comments.
2) msgid "" which identifies the header of PO file gets populated with parts of multi-line captions when auto-updaing PO file, appending more and more data on every update (try re-saving the form multiple times).
Tagsi18n, localization, LRT, PO, translation
Fixed in Revision52679, 52688, 52695, 52696
LazTarget-
Widgetset
Attached Files
  • Example-New-Lines-Break-LRT-PO.zip (3,811 bytes)
  • lazarus.translations.patch (11,039 bytes)
    Index: ide/idetranslations.pas
    ===================================================================
    --- ide/idetranslations.pas	(revision 51634)
    +++ ide/idetranslations.pas	(working copy)
    @@ -235,7 +235,7 @@
         for i:=0 to Files.Count-1 do begin
           RSTFilename:=RSTDirectory+Files[i];
           Ext:=LowerCase(ExtractFileExt(RSTFilename));
    -      if (Ext<>'.rst') and (Ext<>'.lrt') and (Ext<>'.rsj') then
    +      if (Ext<>'.rst') and (Ext<>'.lrt') and (Ext<>'.rsj') and (Ext<>'.lrj') then
             continue;
           if POFilename='' then
             OutputFilename:=PODirectory+ChangeFileExt(Files[i],'.po')
    @@ -258,13 +258,13 @@
           end else begin
             // there is already a source file for this .po file
             //debugln(['ConvertRSTFiles found another source: ',RSTFilename]);
    -        if (Ext='.rsj') or (Ext='.rst') then begin
    +        if (Ext='.rsj') or (Ext='.rst') or (Ext='.lrt') or (Ext='.lrj') then begin
               // rsj are created by FPC 2.7.1+, rst by older => use only the newest
               for j:=Item^.RSTFileList.Count-1 downto 0 do begin
                 OtherRSTFilename:=Item^.RSTFileList[j];
                 //debugln(['ConvertRSTFiles old: ',OtherRSTFilename]);
                 OtherExt:=LowerCase(ExtractFileExt(OtherRSTFilename));
    -            if (OtherExt='.rsj') or (OtherExt='.rst') then begin
    +            if (OtherExt='.rsj') or (OtherExt='.rst') or (OtherExt='.lrt') or (OtherExt='.lrj') then begin
                   if FileAgeCached(RSTFilename)<=FileAgeCached(OtherRSTFilename) then
                   begin
                     // this one is older => skip
    @@ -376,11 +376,13 @@
           BasePOFile.ReadPOText(POBuf.Source);
         BasePOFile.Tag:=1;
     
    -    // Update po file with lrt or/and rst/rsj files
    +    // Update po file with lrt/lrj or/and rst/rsj files
         for i:=0 to SrcFiles.Count-1 do begin
           Filename:=SrcFiles[i];
           if CompareFileExt(Filename,'.lrt',false)=0 then
             FileType:=stLrt
    +      else if CompareFileExt(Filename,'.lrj',false)=0 then
    +        FileType:=stLrj
           else if CompareFileExt(Filename,'.rst',false)=0 then
             FileType:=stRst
           else if CompareFileExt(Filename,'.rsj',false)=0 then
    Index: ide/main.pp
    ===================================================================
    --- ide/main.pp	(revision 51634)
    +++ ide/main.pp	(working copy)
    @@ -4073,6 +4073,7 @@
       POFileAgeValid: Boolean;
       POOutDir: String;
       LRTFilename: String;
    +  LRJFilename: String;
       UnitOutputDir: String;
       RSTFilename: String;
       RSJFilename: String;
    @@ -4122,11 +4123,15 @@
           if (AProject.MainFilename<>CurFilename)
           and (not FilenameIsPascalUnit(CurFilename)) then
             continue;
    -      // check .lrt file
    +      // check .lrt/.lrj file
           LRTFilename:=ChangeFileExt(CurFilename,'.lrt');
    +      LRJFilename:=ChangeFileExt(CurFilename,'.lrj');
           if FileExistsCached(LRTFilename)
           and ((not POFileAgeValid) or (FileAgeCached(LRTFilename)>POFileAge)) then
             Files[LRTFilename]:=nil;
    +      if FileExistsCached(LRJFilename)
    +      and ((not POFileAgeValid) or (FileAgeCached(LRJFilename)>POFileAge)) then
    +        Files[LRJFilename]:=nil;
           // check .rst/.rsj file
           RSTFilename:=ChangeFileExt(CurFilename,'.rst');
           RSJFilename:=ChangeFileExt(CurFilename,'.rsj');
    Index: ide/sourcefilemanager.pas
    ===================================================================
    --- ide/sourcefilemanager.pas	(revision 51634)
    +++ ide/sourcefilemanager.pas	(working copy)
    @@ -47,7 +47,7 @@
       CodeToolsStructs, ConvCodeTool, CodeCache, CodeTree, FindDeclarationTool,
       BasicCodeTools, SynEdit, UnitResources, IDEExternToolIntf, ObjectInspector,
       PublishModule, etMessagesWnd,
    -  FormEditingIntf;
    +  FormEditingIntf, fpjson;
     
     type
     
    @@ -4772,9 +4772,26 @@
     end;
     
     type
    -  TLRTGrubber = class(TObject)
    +  TTranslateStringItem = record
    +    Name: String;
    +    Value: String;
    +  end;
    +
    +  TTranslateStrings = class
       private
    -    FGrubbed: TStrings;
    +    FList: array of TTranslateStringItem;
    +    function CalcHash(const S: string): Cardinal;
    +    function GetSourceBytes(const S: string): string;
    +  public
    +    destructor Destroy; override;
    +    procedure Add(const AName, AValue: String);
    +    function Count: Integer;
    +    function Text: String;
    +  end;
    +
    +  TLRJGrubber = class(TObject)
    +  private
    +    FGrubbed: TTranslateStrings;
         FWriter: TWriter;
       public
         constructor Create(TheWriter: TWriter);
    @@ -4781,25 +4798,102 @@
         destructor Destroy; override;
         procedure Grub(Sender: TObject; const Instance: TPersistent;
                        PropInfo: PPropInfo; var Content: string);
    -    property Grubbed: TStrings read FGrubbed;
    +    property Grubbed: TTranslateStrings read FGrubbed;
         property Writer: TWriter read FWriter write FWriter;
       end;
     
    -constructor TLRTGrubber.Create(TheWriter: TWriter);
    +function TTranslateStrings.CalcHash(const S: string): Cardinal;
    +var
    +  g: Cardinal;
    +  i: Longint;
     begin
    +  Result:=0;
    +  for i:=1 to Length(s) do
    +  begin
    +    Result:=Result shl 4;
    +    inc(Result,Ord(S[i]));
    +    g:=Result and ($f shl 28);
    +    if g<>0 then
    +     begin
    +       Result:=Result xor (g shr 24);
    +       Result:=Result xor g;
    +     end;
    +  end;
    +  If Result=0 then
    +    Result:=$ffffffff;
    +end;
    +
    +function TTranslateStrings.GetSourceBytes(const S: string): string;
    +var
    +  i, l: Integer;
    +begin
    +  Result:='';
    +  l:=Length(S);
    +  for i:=1 to l do
    +  begin
    +    Result:=Result+IntToStr(Ord(S[i]));
    +    if i<>l then
    +     Result:=Result+',';
    +  end;
    +end;
    +
    +destructor TTranslateStrings.Destroy;
    +begin
    +  SetLength(FList,0);
    +end;
    +
    +procedure TTranslateStrings.Add(const AName, AValue: String);
    +begin
    +  SetLength(FList,Length(FList)+1);
    +  with FList[High(FList)] do
    +  begin
    +    Name:=AName;
    +    Value:=AValue;
    +  end;
    +end;
    +
    +function TTranslateStrings.Count: Integer;
    +begin
    +  Result:=Length(FList);
    +end;
    +
    +function TTranslateStrings.Text: String;
    +var
    +  i: Integer;
    +  R: TTranslateStringItem;
    +begin
    +  Result:='';
    +  if Length(FList)=0 then Exit;
    +  Result:='{"version":1,"strings":['+LineEnding;
    +  for i:=Low(FList) to High(FList) do
    +  begin
    +    R:=TTranslateStringItem(FList[i]);
    +    Result:=Result+'{"hash":'+IntToStr(CalcHash(R.Value))+',"name":"'+R.Name+
    +      '","sourcebytes":['+GetSourceBytes(R.Value)+
    +      '],"value":"'+StringToJSONString(R.Value)+'"}';
    +    if i<High(FList) then
    +      Result:=Result+','+LineEnding
    +    else
    +      Result:=Result+LineEnding;
    +  end;
    +  Result:=Result+']}'+LineEnding;
    +end;
    +
    +constructor TLRJGrubber.Create(TheWriter: TWriter);
    +begin
       inherited Create;
    -  FGrubbed:=TStringList.Create;
    +  FGrubbed:=TTranslateStrings.Create;
       FWriter:=TheWriter;
       FWriter.OnWriteStringProperty:=@Grub;
     end;
     
    -destructor TLRTGrubber.Destroy;
    +destructor TLRJGrubber.Destroy;
     begin
       FGrubbed.Free;
       inherited Destroy;
     end;
     
    -procedure TLRTGrubber.Grub(Sender: TObject; const Instance: TPersistent;
    +procedure TLRJGrubber.Grub(Sender: TObject; const Instance: TPersistent;
       PropInfo: PPropInfo; var Content: string);
     var
       LRSWriter: TLRSObjectWriter;
    @@ -4814,8 +4908,7 @@
       end else begin
         Path:=Instance.ClassName+'.'+PropInfo^.Name;
       end;
    -  FGrubbed.Add(Uppercase(Path)+'='+Content);
    -  //DebugLn(['TLRTGrubber.Grub "',FGrubbed[FGrubbed.Count-1],'"']);
    +  FGrubbed.Add(LowerCase(Path),Content);
     end;
     
     function TLazSourceFileManager.SaveUnitComponent(AnUnitInfo: TUnitInfo;
    @@ -4859,8 +4952,8 @@
       ACaption, AText: string;
       CompResourceCode, LFMFilename, TestFilename: string;
       ADesigner: TIDesigner;
    -  Grubber: TLRTGrubber;
    -  LRTFilename: String;
    +  Grubber: TLRJGrubber;
    +  LRJFilename: String;
       AncestorUnit: TUnitInfo;
       Ancestor: TComponent;
       HasI18N: Boolean;
    @@ -4919,10 +5012,10 @@
             try
               BinCompStream.Position:=0;
               Writer:=AnUnitInfo.UnitResourceFileformat.CreateWriter(BinCompStream,DestroyDriver);
    -          // used to save lrt files
    +          // used to save lrj files
               HasI18N:=IsI18NEnabled(UnitOwners);
               if HasI18N then
    -            Grubber:=TLRTGrubber.Create(Writer);
    +            Grubber:=TLRJGrubber.Create(Writer);
               Writer.OnWriteMethodProperty:=@FormEditor1.WriteMethodPropertyEvent;
               //DebugLn(['TLazSourceFileManager.SaveUnitComponent AncestorInstance=',dbgsName(AncestorInstance)]);
               Writer.OnFindAncestor:=@FormEditor1.WriterFindAncestor;
    @@ -5091,15 +5184,15 @@
           // Now the most important file (.lfm) is saved.
           // Now save the secondary files
     
    -      // save the .lrt file containing the list of all translatable strings of
    +      // save the .lrj file containing the list of all translatable strings of
           // the component
           if ComponentSavingOk
           and (Grubber<>nil) and (Grubber.Grubbed.Count>0)
           and (not (sfSaveToTestDir in Flags))
           and (not AnUnitInfo.IsVirtual) then begin
    -        LRTFilename:=ChangeFileExt(AnUnitInfo.Filename,'.lrt');
    -        DebugLn(['TLazSourceFileManager.SaveUnitComponent save lrt: ',LRTFilename]);
    -        Result:=SaveStringToFile(LRTFilename,Grubber.Grubbed.Text,
    +        LRJFilename:=ChangeFileExt(AnUnitInfo.Filename,'.lrj');
    +        DebugLn(['TLazSourceFileManager.SaveUnitComponent save lrj: ',LRJFilename]);
    +        Result:=SaveStringToFile(LRJFilename,Grubber.Grubbed.Text,
                                      [mbIgnore,mbAbort],AnUnitInfo.Filename);
             if (Result<>mrOk) and (Result<>mrIgnore) then exit;
           end;
    Index: lcl/translations.pas
    ===================================================================
    --- lcl/translations.pas	(revision 51634)
    +++ lcl/translations.pas	(working copy)
    @@ -93,6 +93,7 @@
     type
       TStringsType = (
         stLrt, // Lazarus resource string table
    +    stLrj, // Lazarus resource String table in JSON format
         stRst, // FPC resource string table (before FPC 2.7.1)
         stRsj  // FPC resource string table in JSON format (since FPC 2.7.1)
         );
    @@ -534,10 +535,11 @@
           BasePOFile := TPOFile.Create;
         BasePOFile.Tag:=1;
     
    -    // Update po file with lrt,rst/rsj of RSTFiles
    +    // Update po file with lrt/lrj,rst/rsj of RSTFiles
         for i:=0 to RSTFiles.Count-1 do begin
           Filename:=RSTFiles[i];
           if (CompareFileExt(Filename,'.lrt')=0) or
    +         (CompareFileExt(Filename,'.lrj')=0) or
              (CompareFileExt(Filename,'.rst')=0) or
              (CompareFileExt(Filename,'.rsj')=0) then
             try
    @@ -549,6 +551,9 @@
               if CompareFileExt(Filename,'.lrt')=0 then
                 BasePOFile.UpdateStrings(InputLines, stLrt)
               else
    +          if CompareFileExt(Filename,'.lrj')=0 then
    +            BasePOFile.UpdateStrings(InputLines, stLrj)
    +          else
               if CompareFileExt(Filename,'.rsj')=0 then
                 BasePOFile.UpdateStrings(InputLines, stRsj)
               else
    @@ -1404,8 +1409,8 @@
     begin
       ClearModuleList;
       UntagAll;
    -  if SType = stRsj then
    -    // .rsj file
    +  if (SType = stLrj) or (SType = stRsj) then
    +    // .lrj/.rsj file
         UpdateFromRSJ
       else
       begin
    
  • sourcefilemanager_excludetmenuitem.pas.patch (613 bytes)
    Index: ide/sourcefilemanager.pas
    ===================================================================
    --- ide/sourcefilemanager.pas	(revision 52679)
    +++ ide/sourcefilemanager.pas	(working copy)
    @@ -4930,6 +4930,7 @@
       if not Assigned(Instance) then exit;
       if not Assigned(PropInfo) then exit;
       if SysUtils.CompareText(PropInfo^.PropType^.Name,'TTRANSLATESTRING')<>0 then exit;
    +  if (SysUtils.CompareText(Instance.ClassName,'TMENUITEM')=0) and (Content='-') then exit;
       if Writer.Driver is TLRSObjectWriter then begin
         LRSWriter:=TLRSObjectWriter(Writer.Driver);
         Path:=LRSWriter.GetStackPath;
    
  • lcltranslator_removelastlineending.pas.patch (1,168 bytes)
    Index: lcl/lcltranslator.pas
    ===================================================================
    --- lcl/lcltranslator.pas	(revision 52679)
    +++ lcl/lcltranslator.pas	(working copy)
    @@ -320,6 +320,15 @@
       TmpStr: string;
       APersistentProp: TPersistent;
       StoreStackPath: string;
    +
    +  procedure NormalizeValue;
    +  begin
    +    // delete last line ending
    +    if Length(TmpStr)>2 then
    +      if RightStr(TmpStr,2)=LineEnding then
    +        SetLength(TmpStr,Length(TmpStr)-2);
    +  end;
    +
     begin
       APropCount := GetPropList(AnInstance.ClassInfo, APropList);
       try
    @@ -336,7 +345,10 @@
                           TmpStr := '';
                           LRSTranslator.TranslateStringProperty(self,aninstance,APropInfo,TmpStr);
                           if TmpStr <>'' then
    -                        SetStrProp(AnInstance, APropInfo, TmpStr);
    +                        begin
    +                          NormalizeValue;
    +                          SetStrProp(AnInstance, APropInfo, TmpStr);
    +                        end;
                           end;
               tkclass:    begin
                           APersistentProp := TPersistent(GetObjectProp(AnInstance, APropInfo, TPersistent));
    

Relationships

has duplicate 0028128 resolvedJuha Manninen Patches Translation, multiline and menuitem 
has duplicate 0028740 resolvedMaxim Ganetsky Lazarus MultiLIne TTranslatestring not written as multiline to .po file 
has duplicate 0028105 resolvedJuha Manninen Lazarus Localization: Multi-line in caption or hint didn't saved right to po file. 
has duplicate 0030068 closedMaxim Ganetsky Lazarus Multiline hints make .po file increasing after every form saving 

Activities

Denis Kozlov

2014-08-02 13:10

reporter  

Example-New-Lines-Break-LRT-PO.zip (3,811 bytes)

Denis Kozlov

2014-08-02 13:11

reporter   ~0076423

Example project is attached, with a form at 2 labels: with and without new line characters. Inspect auto-generated LRT and PO files to see the problems.

Maxim Ganetsky

2014-10-10 00:27

developer   ~0078124

Confirmed.

Maxim Ganetsky

2016-02-11 12:36

developer   ~0089955

Last edited: 2016-02-11 12:36

View 2 revisions

Just for the record.

The proper way to solve this bug is to emit RSJ file (like FPC 3.0.0 and up) instead of LRT.
This should solve all such problems and won't require any changes to translations.pas (aside from LRT handling logic cleanup).

Nur Cholif Murtadho

2016-02-15 18:46

reporter   ~0090037

How about .lrj file for lazarus?

Nur Cholif Murtadho

2016-02-15 18:46

reporter  

lazarus.translations.patch (11,039 bytes)
Index: ide/idetranslations.pas
===================================================================
--- ide/idetranslations.pas	(revision 51634)
+++ ide/idetranslations.pas	(working copy)
@@ -235,7 +235,7 @@
     for i:=0 to Files.Count-1 do begin
       RSTFilename:=RSTDirectory+Files[i];
       Ext:=LowerCase(ExtractFileExt(RSTFilename));
-      if (Ext<>'.rst') and (Ext<>'.lrt') and (Ext<>'.rsj') then
+      if (Ext<>'.rst') and (Ext<>'.lrt') and (Ext<>'.rsj') and (Ext<>'.lrj') then
         continue;
       if POFilename='' then
         OutputFilename:=PODirectory+ChangeFileExt(Files[i],'.po')
@@ -258,13 +258,13 @@
       end else begin
         // there is already a source file for this .po file
         //debugln(['ConvertRSTFiles found another source: ',RSTFilename]);
-        if (Ext='.rsj') or (Ext='.rst') then begin
+        if (Ext='.rsj') or (Ext='.rst') or (Ext='.lrt') or (Ext='.lrj') then begin
           // rsj are created by FPC 2.7.1+, rst by older => use only the newest
           for j:=Item^.RSTFileList.Count-1 downto 0 do begin
             OtherRSTFilename:=Item^.RSTFileList[j];
             //debugln(['ConvertRSTFiles old: ',OtherRSTFilename]);
             OtherExt:=LowerCase(ExtractFileExt(OtherRSTFilename));
-            if (OtherExt='.rsj') or (OtherExt='.rst') then begin
+            if (OtherExt='.rsj') or (OtherExt='.rst') or (OtherExt='.lrt') or (OtherExt='.lrj') then begin
               if FileAgeCached(RSTFilename)<=FileAgeCached(OtherRSTFilename) then
               begin
                 // this one is older => skip
@@ -376,11 +376,13 @@
       BasePOFile.ReadPOText(POBuf.Source);
     BasePOFile.Tag:=1;
 
-    // Update po file with lrt or/and rst/rsj files
+    // Update po file with lrt/lrj or/and rst/rsj files
     for i:=0 to SrcFiles.Count-1 do begin
       Filename:=SrcFiles[i];
       if CompareFileExt(Filename,'.lrt',false)=0 then
         FileType:=stLrt
+      else if CompareFileExt(Filename,'.lrj',false)=0 then
+        FileType:=stLrj
       else if CompareFileExt(Filename,'.rst',false)=0 then
         FileType:=stRst
       else if CompareFileExt(Filename,'.rsj',false)=0 then
Index: ide/main.pp
===================================================================
--- ide/main.pp	(revision 51634)
+++ ide/main.pp	(working copy)
@@ -4073,6 +4073,7 @@
   POFileAgeValid: Boolean;
   POOutDir: String;
   LRTFilename: String;
+  LRJFilename: String;
   UnitOutputDir: String;
   RSTFilename: String;
   RSJFilename: String;
@@ -4122,11 +4123,15 @@
       if (AProject.MainFilename<>CurFilename)
       and (not FilenameIsPascalUnit(CurFilename)) then
         continue;
-      // check .lrt file
+      // check .lrt/.lrj file
       LRTFilename:=ChangeFileExt(CurFilename,'.lrt');
+      LRJFilename:=ChangeFileExt(CurFilename,'.lrj');
       if FileExistsCached(LRTFilename)
       and ((not POFileAgeValid) or (FileAgeCached(LRTFilename)>POFileAge)) then
         Files[LRTFilename]:=nil;
+      if FileExistsCached(LRJFilename)
+      and ((not POFileAgeValid) or (FileAgeCached(LRJFilename)>POFileAge)) then
+        Files[LRJFilename]:=nil;
       // check .rst/.rsj file
       RSTFilename:=ChangeFileExt(CurFilename,'.rst');
       RSJFilename:=ChangeFileExt(CurFilename,'.rsj');
Index: ide/sourcefilemanager.pas
===================================================================
--- ide/sourcefilemanager.pas	(revision 51634)
+++ ide/sourcefilemanager.pas	(working copy)
@@ -47,7 +47,7 @@
   CodeToolsStructs, ConvCodeTool, CodeCache, CodeTree, FindDeclarationTool,
   BasicCodeTools, SynEdit, UnitResources, IDEExternToolIntf, ObjectInspector,
   PublishModule, etMessagesWnd,
-  FormEditingIntf;
+  FormEditingIntf, fpjson;
 
 type
 
@@ -4772,9 +4772,26 @@
 end;
 
 type
-  TLRTGrubber = class(TObject)
+  TTranslateStringItem = record
+    Name: String;
+    Value: String;
+  end;
+
+  TTranslateStrings = class
   private
-    FGrubbed: TStrings;
+    FList: array of TTranslateStringItem;
+    function CalcHash(const S: string): Cardinal;
+    function GetSourceBytes(const S: string): string;
+  public
+    destructor Destroy; override;
+    procedure Add(const AName, AValue: String);
+    function Count: Integer;
+    function Text: String;
+  end;
+
+  TLRJGrubber = class(TObject)
+  private
+    FGrubbed: TTranslateStrings;
     FWriter: TWriter;
   public
     constructor Create(TheWriter: TWriter);
@@ -4781,25 +4798,102 @@
     destructor Destroy; override;
     procedure Grub(Sender: TObject; const Instance: TPersistent;
                    PropInfo: PPropInfo; var Content: string);
-    property Grubbed: TStrings read FGrubbed;
+    property Grubbed: TTranslateStrings read FGrubbed;
     property Writer: TWriter read FWriter write FWriter;
   end;
 
-constructor TLRTGrubber.Create(TheWriter: TWriter);
+function TTranslateStrings.CalcHash(const S: string): Cardinal;
+var
+  g: Cardinal;
+  i: Longint;
 begin
+  Result:=0;
+  for i:=1 to Length(s) do
+  begin
+    Result:=Result shl 4;
+    inc(Result,Ord(S[i]));
+    g:=Result and ($f shl 28);
+    if g<>0 then
+     begin
+       Result:=Result xor (g shr 24);
+       Result:=Result xor g;
+     end;
+  end;
+  If Result=0 then
+    Result:=$ffffffff;
+end;
+
+function TTranslateStrings.GetSourceBytes(const S: string): string;
+var
+  i, l: Integer;
+begin
+  Result:='';
+  l:=Length(S);
+  for i:=1 to l do
+  begin
+    Result:=Result+IntToStr(Ord(S[i]));
+    if i<>l then
+     Result:=Result+',';
+  end;
+end;
+
+destructor TTranslateStrings.Destroy;
+begin
+  SetLength(FList,0);
+end;
+
+procedure TTranslateStrings.Add(const AName, AValue: String);
+begin
+  SetLength(FList,Length(FList)+1);
+  with FList[High(FList)] do
+  begin
+    Name:=AName;
+    Value:=AValue;
+  end;
+end;
+
+function TTranslateStrings.Count: Integer;
+begin
+  Result:=Length(FList);
+end;
+
+function TTranslateStrings.Text: String;
+var
+  i: Integer;
+  R: TTranslateStringItem;
+begin
+  Result:='';
+  if Length(FList)=0 then Exit;
+  Result:='{"version":1,"strings":['+LineEnding;
+  for i:=Low(FList) to High(FList) do
+  begin
+    R:=TTranslateStringItem(FList[i]);
+    Result:=Result+'{"hash":'+IntToStr(CalcHash(R.Value))+',"name":"'+R.Name+
+      '","sourcebytes":['+GetSourceBytes(R.Value)+
+      '],"value":"'+StringToJSONString(R.Value)+'"}';
+    if i<High(FList) then
+      Result:=Result+','+LineEnding
+    else
+      Result:=Result+LineEnding;
+  end;
+  Result:=Result+']}'+LineEnding;
+end;
+
+constructor TLRJGrubber.Create(TheWriter: TWriter);
+begin
   inherited Create;
-  FGrubbed:=TStringList.Create;
+  FGrubbed:=TTranslateStrings.Create;
   FWriter:=TheWriter;
   FWriter.OnWriteStringProperty:=@Grub;
 end;
 
-destructor TLRTGrubber.Destroy;
+destructor TLRJGrubber.Destroy;
 begin
   FGrubbed.Free;
   inherited Destroy;
 end;
 
-procedure TLRTGrubber.Grub(Sender: TObject; const Instance: TPersistent;
+procedure TLRJGrubber.Grub(Sender: TObject; const Instance: TPersistent;
   PropInfo: PPropInfo; var Content: string);
 var
   LRSWriter: TLRSObjectWriter;
@@ -4814,8 +4908,7 @@
   end else begin
     Path:=Instance.ClassName+'.'+PropInfo^.Name;
   end;
-  FGrubbed.Add(Uppercase(Path)+'='+Content);
-  //DebugLn(['TLRTGrubber.Grub "',FGrubbed[FGrubbed.Count-1],'"']);
+  FGrubbed.Add(LowerCase(Path),Content);
 end;
 
 function TLazSourceFileManager.SaveUnitComponent(AnUnitInfo: TUnitInfo;
@@ -4859,8 +4952,8 @@
   ACaption, AText: string;
   CompResourceCode, LFMFilename, TestFilename: string;
   ADesigner: TIDesigner;
-  Grubber: TLRTGrubber;
-  LRTFilename: String;
+  Grubber: TLRJGrubber;
+  LRJFilename: String;
   AncestorUnit: TUnitInfo;
   Ancestor: TComponent;
   HasI18N: Boolean;
@@ -4919,10 +5012,10 @@
         try
           BinCompStream.Position:=0;
           Writer:=AnUnitInfo.UnitResourceFileformat.CreateWriter(BinCompStream,DestroyDriver);
-          // used to save lrt files
+          // used to save lrj files
           HasI18N:=IsI18NEnabled(UnitOwners);
           if HasI18N then
-            Grubber:=TLRTGrubber.Create(Writer);
+            Grubber:=TLRJGrubber.Create(Writer);
           Writer.OnWriteMethodProperty:=@FormEditor1.WriteMethodPropertyEvent;
           //DebugLn(['TLazSourceFileManager.SaveUnitComponent AncestorInstance=',dbgsName(AncestorInstance)]);
           Writer.OnFindAncestor:=@FormEditor1.WriterFindAncestor;
@@ -5091,15 +5184,15 @@
       // Now the most important file (.lfm) is saved.
       // Now save the secondary files
 
-      // save the .lrt file containing the list of all translatable strings of
+      // save the .lrj file containing the list of all translatable strings of
       // the component
       if ComponentSavingOk
       and (Grubber<>nil) and (Grubber.Grubbed.Count>0)
       and (not (sfSaveToTestDir in Flags))
       and (not AnUnitInfo.IsVirtual) then begin
-        LRTFilename:=ChangeFileExt(AnUnitInfo.Filename,'.lrt');
-        DebugLn(['TLazSourceFileManager.SaveUnitComponent save lrt: ',LRTFilename]);
-        Result:=SaveStringToFile(LRTFilename,Grubber.Grubbed.Text,
+        LRJFilename:=ChangeFileExt(AnUnitInfo.Filename,'.lrj');
+        DebugLn(['TLazSourceFileManager.SaveUnitComponent save lrj: ',LRJFilename]);
+        Result:=SaveStringToFile(LRJFilename,Grubber.Grubbed.Text,
                                  [mbIgnore,mbAbort],AnUnitInfo.Filename);
         if (Result<>mrOk) and (Result<>mrIgnore) then exit;
       end;
Index: lcl/translations.pas
===================================================================
--- lcl/translations.pas	(revision 51634)
+++ lcl/translations.pas	(working copy)
@@ -93,6 +93,7 @@
 type
   TStringsType = (
     stLrt, // Lazarus resource string table
+    stLrj, // Lazarus resource String table in JSON format
     stRst, // FPC resource string table (before FPC 2.7.1)
     stRsj  // FPC resource string table in JSON format (since FPC 2.7.1)
     );
@@ -534,10 +535,11 @@
       BasePOFile := TPOFile.Create;
     BasePOFile.Tag:=1;
 
-    // Update po file with lrt,rst/rsj of RSTFiles
+    // Update po file with lrt/lrj,rst/rsj of RSTFiles
     for i:=0 to RSTFiles.Count-1 do begin
       Filename:=RSTFiles[i];
       if (CompareFileExt(Filename,'.lrt')=0) or
+         (CompareFileExt(Filename,'.lrj')=0) or
          (CompareFileExt(Filename,'.rst')=0) or
          (CompareFileExt(Filename,'.rsj')=0) then
         try
@@ -549,6 +551,9 @@
           if CompareFileExt(Filename,'.lrt')=0 then
             BasePOFile.UpdateStrings(InputLines, stLrt)
           else
+          if CompareFileExt(Filename,'.lrj')=0 then
+            BasePOFile.UpdateStrings(InputLines, stLrj)
+          else
           if CompareFileExt(Filename,'.rsj')=0 then
             BasePOFile.UpdateStrings(InputLines, stRsj)
           else
@@ -1404,8 +1409,8 @@
 begin
   ClearModuleList;
   UntagAll;
-  if SType = stRsj then
-    // .rsj file
+  if (SType = stLrj) or (SType = stRsj) then
+    // .lrj/.rsj file
     UpdateFromRSJ
   else
   begin

Maxim Ganetsky

2016-02-15 23:20

developer   ~0090041

Why .lrj and not .rsj?

Nur Cholif Murtadho

2016-02-16 02:47

reporter   ~0090046

Do you mean merging it with the one generated by compiler in output directory?
It's always rewritten for each compile. And the filename may overlap each other if output directory is the same with project directory. Isn't lazarus and fpc resource string table designed to be separated?

We have .lrt and .rst before, and then .rsj for json format. I just assume the lazarus string table in json would be .lrj

TStringsType = (
    stLrt, // Lazarus resource string table
    stLrj, // Lazarus resource String table in JSON format
    stRst, // FPC resource string table (before FPC 2.7.1)
    stRsj // FPC resource string table in JSON format (since FPC 2.7.1)
    );

Maxim Ganetsky

2016-02-16 12:42

developer   ~0090059

OK, point taken.

Maxim Ganetsky

2016-07-14 01:37

developer   ~0093694

Last edited: 2016-07-14 01:39

View 3 revisions

Please test and close if OK.

Keep in mind that you need to run "Project" -> "Resave forms with enabled i18n" menu item in order to generate LRJ files.

@riderkick: what is your name? I will add you to contributors list.

Nur Cholif Murtadho

2016-07-14 08:58

reporter   ~0093697

It seems OK.
I just added my real name.

There is some changes left on my copy of lazarus when I update to the latest revision. Maybe you can consider to add them.
On TMenuItem if we us "-" that will make separator. Maybe you can consider to exclude them to be saved to po file. This item shouldn't be included in po file, because they shouldn't be changed, and I think no one will ever translate this item.
Also there is a bug(?) when we use multiline in translation it will add extra lineending. It's due to po format that required(?) to use \n at the end of multilined translation item. So when loading the po file, it will add extra lineending.

Nur Cholif Murtadho

2016-07-14 08:58

reporter  

sourcefilemanager_excludetmenuitem.pas.patch (613 bytes)
Index: ide/sourcefilemanager.pas
===================================================================
--- ide/sourcefilemanager.pas	(revision 52679)
+++ ide/sourcefilemanager.pas	(working copy)
@@ -4930,6 +4930,7 @@
   if not Assigned(Instance) then exit;
   if not Assigned(PropInfo) then exit;
   if SysUtils.CompareText(PropInfo^.PropType^.Name,'TTRANSLATESTRING')<>0 then exit;
+  if (SysUtils.CompareText(Instance.ClassName,'TMENUITEM')=0) and (Content='-') then exit;
   if Writer.Driver is TLRSObjectWriter then begin
     LRSWriter:=TLRSObjectWriter(Writer.Driver);
     Path:=LRSWriter.GetStackPath;

Nur Cholif Murtadho

2016-07-14 08:58

reporter  

lcltranslator_removelastlineending.pas.patch (1,168 bytes)
Index: lcl/lcltranslator.pas
===================================================================
--- lcl/lcltranslator.pas	(revision 52679)
+++ lcl/lcltranslator.pas	(working copy)
@@ -320,6 +320,15 @@
   TmpStr: string;
   APersistentProp: TPersistent;
   StoreStackPath: string;
+
+  procedure NormalizeValue;
+  begin
+    // delete last line ending
+    if Length(TmpStr)>2 then
+      if RightStr(TmpStr,2)=LineEnding then
+        SetLength(TmpStr,Length(TmpStr)-2);
+  end;
+
 begin
   APropCount := GetPropList(AnInstance.ClassInfo, APropList);
   try
@@ -336,7 +345,10 @@
                       TmpStr := '';
                       LRSTranslator.TranslateStringProperty(self,aninstance,APropInfo,TmpStr);
                       if TmpStr <>'' then
-                        SetStrProp(AnInstance, APropInfo, TmpStr);
+                        begin
+                          NormalizeValue;
+                          SetStrProp(AnInstance, APropInfo, TmpStr);
+                        end;
                       end;
           tkclass:    begin
                       APersistentProp := TPersistent(GetObjectProp(AnInstance, APropInfo, TPersistent));

Maxim Ganetsky

2016-07-14 13:41

developer   ~0093707

Patches seem good, I'll look at them. I added you to contributors list.

Maxim Ganetsky

2016-07-16 01:15

developer   ~0093729

@Nur Cholif Murtadho.

Your menuitem exclusion patch is applied, thanks.
Your lineending patch was not applied, because more general solution was implemented based on its idea in r52696. This solution should work even when resourcestring was assigned manually.

Please test. If there are any issues not related to original topic of this bug, please create separate bug reports.

Resolving as fixed.

Nur Cholif Murtadho

2016-07-16 08:43

reporter   ~0093730

All seems good, thanks.

Issue History

Date Modified Username Field Change
2014-08-02 13:05 Denis Kozlov New Issue
2014-08-02 13:06 Denis Kozlov Tag Attached: i18n
2014-08-02 13:06 Denis Kozlov Tag Attached: LRT
2014-08-02 13:06 Denis Kozlov Tag Attached: PO
2014-08-02 13:06 Denis Kozlov Tag Attached: translation
2014-08-02 13:06 Denis Kozlov Tag Attached: localization
2014-08-02 13:10 Denis Kozlov File Added: Example-New-Lines-Break-LRT-PO.zip
2014-08-02 13:11 Denis Kozlov Note Added: 0076423
2014-08-19 22:38 Maxim Ganetsky Relationship added related to 0026616
2014-10-10 00:26 Maxim Ganetsky Relationship deleted related to 0026616
2014-10-10 00:27 Maxim Ganetsky LazTarget => -
2014-10-10 00:27 Maxim Ganetsky Note Added: 0078124
2014-10-10 00:27 Maxim Ganetsky Assigned To => Maxim Ganetsky
2014-10-10 00:27 Maxim Ganetsky Status new => confirmed
2014-10-10 00:27 Maxim Ganetsky Assigned To Maxim Ganetsky =>
2015-05-18 23:11 Maxim Ganetsky Relationship added related to 0028128
2015-09-28 13:07 Maxim Ganetsky Relationship added related to 0028740
2016-02-10 08:29 Juha Manninen Relationship added has duplicate 0028105
2016-02-10 08:31 Juha Manninen Relationship replaced has duplicate 0028128
2016-02-11 12:36 Maxim Ganetsky Note Added: 0089955
2016-02-11 12:36 Maxim Ganetsky Note Edited: 0089955 View Revisions
2016-02-15 18:46 Nur Cholif Murtadho Note Added: 0090037
2016-02-15 18:46 Nur Cholif Murtadho File Added: lazarus.translations.patch
2016-02-15 23:20 Maxim Ganetsky Note Added: 0090041
2016-02-16 02:47 Nur Cholif Murtadho Note Added: 0090046
2016-02-16 12:41 Maxim Ganetsky Assigned To => Maxim Ganetsky
2016-02-16 12:41 Maxim Ganetsky Status confirmed => assigned
2016-02-16 12:42 Maxim Ganetsky Note Added: 0090059
2016-02-16 12:46 Maxim Ganetsky Relationship replaced has duplicate 0028740
2016-04-27 00:19 Maxim Ganetsky Relationship added has duplicate 0030068
2016-07-14 01:37 Maxim Ganetsky Fixed in Revision => 52679
2016-07-14 01:37 Maxim Ganetsky Note Added: 0093694
2016-07-14 01:37 Maxim Ganetsky Status assigned => resolved
2016-07-14 01:37 Maxim Ganetsky Fixed in Version => 1.8
2016-07-14 01:37 Maxim Ganetsky Resolution open => fixed
2016-07-14 01:39 Maxim Ganetsky Note Edited: 0093694 View Revisions
2016-07-14 01:39 Maxim Ganetsky Note Edited: 0093694 View Revisions
2016-07-14 08:58 Nur Cholif Murtadho Note Added: 0093697
2016-07-14 08:58 Nur Cholif Murtadho File Added: sourcefilemanager_excludetmenuitem.pas.patch
2016-07-14 08:58 Nur Cholif Murtadho File Added: lcltranslator_removelastlineending.pas.patch
2016-07-14 13:35 Maxim Ganetsky Status resolved => assigned
2016-07-14 13:35 Maxim Ganetsky Resolution fixed => reopened
2016-07-14 13:41 Maxim Ganetsky Note Added: 0093707
2016-07-16 01:15 Maxim Ganetsky Fixed in Revision 52679 => 52679, 52688, 52695, 52696
2016-07-16 01:15 Maxim Ganetsky Note Added: 0093729
2016-07-16 01:15 Maxim Ganetsky Status assigned => resolved
2016-07-16 01:15 Maxim Ganetsky Resolution reopened => fixed
2016-07-16 08:43 Nur Cholif Murtadho Note Added: 0093730