View Issue Details

IDProjectCategoryView StatusLast Update
0031385FPCPackagespublic2019-09-11 13:51
ReporterDo-wan KimAssigned ToMichael Van Canneyt 
PrioritynormalSeverityminorReproducibilityalways
Status closedResolutionfixed 
Platformx86OSWindowsOS Version10
Product Version3.1.1Product Build35433 
Target VersionFixed in Version3.3.1 
Summary0031385: memds more friendly with utf-8.
Descriptionmemds can store utf-8 string, but locate and lookup cannot work with utf-8 due to VarArray cannot support varString variant type and change encoding to olestr.

simple patch work good with utf-8 string.




TagsNo tags attached.
Fixed in Revision42974
FPCOldBugId
FPCTarget3.2.0
Attached Files
  • memds.pp_utf8_support.patch (2,024 bytes)
    Index: packages/fcl-db/src/memds/memds.pp
    ===================================================================
    --- packages/fcl-db/src/memds/memds.pp	(revision 35434)
    +++ packages/fcl-db/src/memds/memds.pp	(working copy)
    @@ -1217,6 +1217,8 @@
       i: integer;
       AField: TField;
       s1,s2: string;
    +  UseUTF8:Boolean;
    +  keyValuesType:array of tvartype;
     begin
       Result := false;
       SaveState := SetTempState(dsFilter);
    @@ -1228,11 +1230,19 @@
           begin
           Matched := lstKeyFields.Count = 1;
           AKeyValues := VarArrayOf([KeyValues]);
    +      SetLength(keyValuesType,1);
    +      keyValuesType[0]:=VarType(KeyValues);
           end
         else if VarArrayDimCount(KeyValues) = 1 then
           begin
           Matched := VarArrayHighBound(KeyValues,1) + 1 = lstKeyFields.Count;
           AKeyValues := KeyValues;
    +      if Matched then
    +        begin
    +          SetLength(keyValuesType,VarArrayHighBound(KeyValues,1) + 1);
    +          for i:=0 to VarArrayHighBound(KeyValues,1) do
    +            keyValuesType[i]:=VarType(keyValuesType[i]);
    +        end;
           end
         else
           Matched := false;
    @@ -1255,12 +1265,19 @@
               // string fields
               if AField.DataType in [ftString, ftFixedChar] then
               begin
    +            UseUTF8:=keyValuesType[i]<>varstring;
                 s1 := AField.AsString;
    +            if UseUTF8 then
    +               s2:=pchar(UTF8Encode(VarToUnicodeStr(AKeyValues[i])))
    +               else
                 s2 := VarToStr(AKeyValues[i]);
                 if loPartialKey in Options then
                   s1 := copy(s1, 1, length(s2));
                 if loCaseInsensitive in Options then
    -              Result := AnsiCompareText(s1, s2)=0
    +              if UseUTF8 then
    +              Result := CompareText(s1, s2)=0
    +                else
    +                  Result:=AnsiCompareText(s1, s2)=0
                 else
                   Result := s1=s2;
               end
    @@ -1277,6 +1294,7 @@
       finally
         lstKeyFields.Free;
         RestoreState(SaveState);
    +    SetLength(keyValuesType,0);
       end;
     end;
     
    
  • test_memds.zip (72,408 bytes)
  • memds.pp_utf8_support_1.patch (2,020 bytes)
    Index: packages/fcl-db/src/memds/memds.pp
    ===================================================================
    --- packages/fcl-db/src/memds/memds.pp	(revision 35434)
    +++ packages/fcl-db/src/memds/memds.pp	(working copy)
    @@ -1217,6 +1217,8 @@
       i: integer;
       AField: TField;
       s1,s2: string;
    +  UseUTF8:Boolean;
    +  keyValuesType:array of tvartype;
     begin
       Result := false;
       SaveState := SetTempState(dsFilter);
    @@ -1228,11 +1230,19 @@
           begin
           Matched := lstKeyFields.Count = 1;
           AKeyValues := VarArrayOf([KeyValues]);
    +      SetLength(keyValuesType,1);
    +      keyValuesType[0]:=VarType(KeyValues);
           end
         else if VarArrayDimCount(KeyValues) = 1 then
           begin
           Matched := VarArrayHighBound(KeyValues,1) + 1 = lstKeyFields.Count;
           AKeyValues := KeyValues;
    +      if Matched then
    +        begin
    +          SetLength(keyValuesType,VarArrayHighBound(KeyValues,1) + 1);
    +          for i:=0 to VarArrayHighBound(KeyValues,1) do
    +            keyValuesType[i]:=VarType(KeyValues[i]);
    +        end;
           end
         else
           Matched := false;
    @@ -1255,12 +1265,19 @@
               // string fields
               if AField.DataType in [ftString, ftFixedChar] then
               begin
    +            UseUTF8:=keyValuesType[i]<>varstring;
                 s1 := AField.AsString;
    +            if UseUTF8 then
    +               s2:=pchar(UTF8Encode(VarToUnicodeStr(AKeyValues[i])))
    +               else
                 s2 := VarToStr(AKeyValues[i]);
                 if loPartialKey in Options then
                   s1 := copy(s1, 1, length(s2));
                 if loCaseInsensitive in Options then
    -              Result := AnsiCompareText(s1, s2)=0
    +              if UseUTF8 then
    +              Result := CompareText(s1, s2)=0
    +                else
    +                  Result:=AnsiCompareText(s1, s2)=0
                 else
                   Result := s1=s2;
               end
    @@ -1277,6 +1294,7 @@
       finally
         lstKeyFields.Free;
         RestoreState(SaveState);
    +    SetLength(keyValuesType,0);
       end;
     end;
     
    
  • test_memds_1.zip (72,408 bytes)
  • memds.pp_updatetolinux.patch (2,325 bytes)
    Index: packages/fcl-db/src/memds/memds.pp
    ===================================================================
    --- packages/fcl-db/src/memds/memds.pp	(revision 35474)
    +++ packages/fcl-db/src/memds/memds.pp	(working copy)
    @@ -1217,6 +1217,10 @@
       i: integer;
       AField: TField;
       s1,s2: string;
    +  {$ifdef WINDOWS}
    +  UseUTF8:Boolean;
    +  keyValuesType:array of tvartype;
    +  {$endif}
     begin
       Result := false;
       SaveState := SetTempState(dsFilter);
    @@ -1228,11 +1232,23 @@
           begin
           Matched := lstKeyFields.Count = 1;
           AKeyValues := VarArrayOf([KeyValues]);
    +      {$ifdef WINDOWS}
    +      SetLength(keyValuesType,1);
    +      keyValuesType[0]:=VarType(KeyValues);
    +      {$endif}
           end
         else if VarArrayDimCount(KeyValues) = 1 then
           begin
           Matched := VarArrayHighBound(KeyValues,1) + 1 = lstKeyFields.Count;
           AKeyValues := KeyValues;
    +      {$ifdef WINDOWS}
    +      if Matched then
    +        begin
    +          SetLength(keyValuesType,VarArrayHighBound(KeyValues,1) + 1);
    +          for i:=0 to VarArrayHighBound(KeyValues,1) do
    +            keyValuesType[i]:=VarType(KeyValues[i]);
    +        end;
    +      {$endif}
           end
         else
           Matched := false;
    @@ -1255,12 +1271,25 @@
               // string fields
               if AField.DataType in [ftString, ftFixedChar] then
               begin
    +            {$ifdef WINDOWS}
    +            UseUTF8:=keyValuesType[i]<>varstring;
    +            {$endif}
                 s1 := AField.AsString;
    +            {$ifdef WINDOWS}
    +            if UseUTF8 then
    +               s2:=pchar(UTF8Encode(VarToUnicodeStr(AKeyValues[i])))
    +               else
    +            {$endif}
                 s2 := VarToStr(AKeyValues[i]);
                 if loPartialKey in Options then
                   s1 := copy(s1, 1, length(s2));
                 if loCaseInsensitive in Options then
    -              Result := AnsiCompareText(s1, s2)=0
    +            {$ifdef WINDOWS}
    +              if UseUTF8 then
    +              Result := CompareText(s1, s2)=0
    +                else
    +            {$endif}
    +                  Result:=AnsiCompareText(s1, s2)=0
                 else
                   Result := s1=s2;
               end
    @@ -1277,6 +1306,9 @@
       finally
         lstKeyFields.Free;
         RestoreState(SaveState);
    +    {$ifdef WINDOWS}
    +    SetLength(keyValuesType,0);
    +    {$endif}
       end;
     end;
     
    

Activities

Do-wan Kim

2017-02-14 15:20

reporter  

memds.pp_utf8_support.patch (2,024 bytes)
Index: packages/fcl-db/src/memds/memds.pp
===================================================================
--- packages/fcl-db/src/memds/memds.pp	(revision 35434)
+++ packages/fcl-db/src/memds/memds.pp	(working copy)
@@ -1217,6 +1217,8 @@
   i: integer;
   AField: TField;
   s1,s2: string;
+  UseUTF8:Boolean;
+  keyValuesType:array of tvartype;
 begin
   Result := false;
   SaveState := SetTempState(dsFilter);
@@ -1228,11 +1230,19 @@
       begin
       Matched := lstKeyFields.Count = 1;
       AKeyValues := VarArrayOf([KeyValues]);
+      SetLength(keyValuesType,1);
+      keyValuesType[0]:=VarType(KeyValues);
       end
     else if VarArrayDimCount(KeyValues) = 1 then
       begin
       Matched := VarArrayHighBound(KeyValues,1) + 1 = lstKeyFields.Count;
       AKeyValues := KeyValues;
+      if Matched then
+        begin
+          SetLength(keyValuesType,VarArrayHighBound(KeyValues,1) + 1);
+          for i:=0 to VarArrayHighBound(KeyValues,1) do
+            keyValuesType[i]:=VarType(keyValuesType[i]);
+        end;
       end
     else
       Matched := false;
@@ -1255,12 +1265,19 @@
           // string fields
           if AField.DataType in [ftString, ftFixedChar] then
           begin
+            UseUTF8:=keyValuesType[i]<>varstring;
             s1 := AField.AsString;
+            if UseUTF8 then
+               s2:=pchar(UTF8Encode(VarToUnicodeStr(AKeyValues[i])))
+               else
             s2 := VarToStr(AKeyValues[i]);
             if loPartialKey in Options then
               s1 := copy(s1, 1, length(s2));
             if loCaseInsensitive in Options then
-              Result := AnsiCompareText(s1, s2)=0
+              if UseUTF8 then
+              Result := CompareText(s1, s2)=0
+                else
+                  Result:=AnsiCompareText(s1, s2)=0
             else
               Result := s1=s2;
           end
@@ -1277,6 +1294,7 @@
   finally
     lstKeyFields.Free;
     RestoreState(SaveState);
+    SetLength(keyValuesType,0);
   end;
 end;
 

Do-wan Kim

2017-02-14 15:23

reporter  

test_memds.zip (72,408 bytes)

Do-wan Kim

2017-02-15 03:26

reporter  

memds.pp_utf8_support_1.patch (2,020 bytes)
Index: packages/fcl-db/src/memds/memds.pp
===================================================================
--- packages/fcl-db/src/memds/memds.pp	(revision 35434)
+++ packages/fcl-db/src/memds/memds.pp	(working copy)
@@ -1217,6 +1217,8 @@
   i: integer;
   AField: TField;
   s1,s2: string;
+  UseUTF8:Boolean;
+  keyValuesType:array of tvartype;
 begin
   Result := false;
   SaveState := SetTempState(dsFilter);
@@ -1228,11 +1230,19 @@
       begin
       Matched := lstKeyFields.Count = 1;
       AKeyValues := VarArrayOf([KeyValues]);
+      SetLength(keyValuesType,1);
+      keyValuesType[0]:=VarType(KeyValues);
       end
     else if VarArrayDimCount(KeyValues) = 1 then
       begin
       Matched := VarArrayHighBound(KeyValues,1) + 1 = lstKeyFields.Count;
       AKeyValues := KeyValues;
+      if Matched then
+        begin
+          SetLength(keyValuesType,VarArrayHighBound(KeyValues,1) + 1);
+          for i:=0 to VarArrayHighBound(KeyValues,1) do
+            keyValuesType[i]:=VarType(KeyValues[i]);
+        end;
       end
     else
       Matched := false;
@@ -1255,12 +1265,19 @@
           // string fields
           if AField.DataType in [ftString, ftFixedChar] then
           begin
+            UseUTF8:=keyValuesType[i]<>varstring;
             s1 := AField.AsString;
+            if UseUTF8 then
+               s2:=pchar(UTF8Encode(VarToUnicodeStr(AKeyValues[i])))
+               else
             s2 := VarToStr(AKeyValues[i]);
             if loPartialKey in Options then
               s1 := copy(s1, 1, length(s2));
             if loCaseInsensitive in Options then
-              Result := AnsiCompareText(s1, s2)=0
+              if UseUTF8 then
+              Result := CompareText(s1, s2)=0
+                else
+                  Result:=AnsiCompareText(s1, s2)=0
             else
               Result := s1=s2;
           end
@@ -1277,6 +1294,7 @@
   finally
     lstKeyFields.Free;
     RestoreState(SaveState);
+    SetLength(keyValuesType,0);
   end;
 end;
 

Do-wan Kim

2017-02-15 03:27

reporter   ~0098178

Last edited: 2017-02-15 03:51

View 2 revisions

I mistype one in patch and fix it.

Do-wan Kim

2017-02-15 03:34

reporter  

test_memds_1.zip (72,408 bytes)

Michael Van Canneyt

2017-02-22 20:34

administrator   ~0098380

I think your test case is flawed.

uses db,memds;

Var
  MemDataset1: TMemDataset;

begin
  MemDataset1:=TMemDataset.Create(Nil);
  try
    MemDataset1.FieldDefs.Add('first',ftString,40);
    MemDataset1.CreateTable;
    MemDataset1.Active:=True;
    MemDataset1.Append;
    MemDataset1.Fields[0].AsString:=#$e2#$99#$af'abcd';
    MemDataset1.Post;
    MemDataset1.Append;
    MemDataset1.Fields[0].AsString:='défaut';
    MemDataset1.Post;
    if MemDataset1.Locate('first',UTF8Decode(#$e2#$99#$af'abcd'),[]) then
      Writeln('ok')
    else
      Writeln('Not ok');
    if MemDataset1.Locate('first','défaut',[]) then
      Writeln('ok')
    else
      Writeln('Not ok');

  finally
    MemDataset1.Free;
  end;
end.


Prints (on linux)

home: >./testl
Not ok
ok

Do-wan Kim

2017-02-23 05:24

reporter   ~0098385

Last edited: 2017-02-23 05:42

View 2 revisions

I got ok, ok under windows.

Problems in windows only, not linux. update patch.

Do-wan Kim

2017-02-23 05:39

reporter  

memds.pp_updatetolinux.patch (2,325 bytes)
Index: packages/fcl-db/src/memds/memds.pp
===================================================================
--- packages/fcl-db/src/memds/memds.pp	(revision 35474)
+++ packages/fcl-db/src/memds/memds.pp	(working copy)
@@ -1217,6 +1217,10 @@
   i: integer;
   AField: TField;
   s1,s2: string;
+  {$ifdef WINDOWS}
+  UseUTF8:Boolean;
+  keyValuesType:array of tvartype;
+  {$endif}
 begin
   Result := false;
   SaveState := SetTempState(dsFilter);
@@ -1228,11 +1232,23 @@
       begin
       Matched := lstKeyFields.Count = 1;
       AKeyValues := VarArrayOf([KeyValues]);
+      {$ifdef WINDOWS}
+      SetLength(keyValuesType,1);
+      keyValuesType[0]:=VarType(KeyValues);
+      {$endif}
       end
     else if VarArrayDimCount(KeyValues) = 1 then
       begin
       Matched := VarArrayHighBound(KeyValues,1) + 1 = lstKeyFields.Count;
       AKeyValues := KeyValues;
+      {$ifdef WINDOWS}
+      if Matched then
+        begin
+          SetLength(keyValuesType,VarArrayHighBound(KeyValues,1) + 1);
+          for i:=0 to VarArrayHighBound(KeyValues,1) do
+            keyValuesType[i]:=VarType(KeyValues[i]);
+        end;
+      {$endif}
       end
     else
       Matched := false;
@@ -1255,12 +1271,25 @@
           // string fields
           if AField.DataType in [ftString, ftFixedChar] then
           begin
+            {$ifdef WINDOWS}
+            UseUTF8:=keyValuesType[i]<>varstring;
+            {$endif}
             s1 := AField.AsString;
+            {$ifdef WINDOWS}
+            if UseUTF8 then
+               s2:=pchar(UTF8Encode(VarToUnicodeStr(AKeyValues[i])))
+               else
+            {$endif}
             s2 := VarToStr(AKeyValues[i]);
             if loPartialKey in Options then
               s1 := copy(s1, 1, length(s2));
             if loCaseInsensitive in Options then
-              Result := AnsiCompareText(s1, s2)=0
+            {$ifdef WINDOWS}
+              if UseUTF8 then
+              Result := CompareText(s1, s2)=0
+                else
+            {$endif}
+                  Result:=AnsiCompareText(s1, s2)=0
             else
               Result := s1=s2;
           end
@@ -1277,6 +1306,9 @@
   finally
     lstKeyFields.Free;
     RestoreState(SaveState);
+    {$ifdef WINDOWS}
+    SetLength(keyValuesType,0);
+    {$endif}
   end;
 end;
 

Michael Van Canneyt

2017-02-23 08:31

administrator   ~0098388

I do not think it has anything to do with Windows, but with the default codepage for single-byte strings. Under Linux, this is UTF8 by default.

Under Windows, this is an Ansi codepage.

So a patch with {$IFDEF SomeOS} in it, is definitely wrong.

What you IMHO should do is look at the new codepage support in TFieldDef and TField and check whether you can use that.

Do-wan Kim

2017-02-24 01:11

reporter   ~0098406

Yes, it make problem under non utf-8 string encoding.

Main problem is variant doesn't support varString type under Windows and it convert to varolestr type.

Michael Van Canneyt

2019-09-11 09:42

administrator   ~0118031

I did some fixes, the new CodePage/Charset support in TStringField allows to handle this without ifdefs. Added your corrected sample as a testcase.

Do-wan Kim

2019-09-11 13:51

reporter   ~0118033

Locate function works ok with UTF8 string. Thank you!

Issue History

Date Modified Username Field Change
2017-02-14 15:20 Do-wan Kim New Issue
2017-02-14 15:20 Do-wan Kim File Added: memds.pp_utf8_support.patch
2017-02-14 15:23 Do-wan Kim File Added: test_memds.zip
2017-02-14 21:47 Michael Van Canneyt Assigned To => Michael Van Canneyt
2017-02-14 21:47 Michael Van Canneyt Status new => assigned
2017-02-15 03:26 Do-wan Kim File Added: memds.pp_utf8_support_1.patch
2017-02-15 03:27 Do-wan Kim Note Added: 0098178
2017-02-15 03:34 Do-wan Kim File Added: test_memds_1.zip
2017-02-15 03:51 Do-wan Kim Note Edited: 0098178 View Revisions
2017-02-22 20:34 Michael Van Canneyt Note Added: 0098380
2017-02-22 20:35 Michael Van Canneyt Status assigned => feedback
2017-02-23 05:24 Do-wan Kim Note Added: 0098385
2017-02-23 05:24 Do-wan Kim Status feedback => assigned
2017-02-23 05:39 Do-wan Kim File Added: memds.pp_updatetolinux.patch
2017-02-23 05:42 Do-wan Kim Note Edited: 0098385 View Revisions
2017-02-23 08:31 Michael Van Canneyt Note Added: 0098388
2017-02-23 08:31 Michael Van Canneyt Status assigned => feedback
2017-02-24 01:11 Do-wan Kim Note Added: 0098406
2017-02-24 01:11 Do-wan Kim Status feedback => assigned
2019-09-11 09:42 Michael Van Canneyt Status assigned => resolved
2019-09-11 09:42 Michael Van Canneyt Resolution open => fixed
2019-09-11 09:42 Michael Van Canneyt Fixed in Version => 3.3.1
2019-09-11 09:42 Michael Van Canneyt Fixed in Revision => 42974
2019-09-11 09:42 Michael Van Canneyt FPCTarget => 3.2.0
2019-09-11 09:42 Michael Van Canneyt Note Added: 0118031
2019-09-11 13:51 Do-wan Kim Status resolved => closed
2019-09-11 13:51 Do-wan Kim Note Added: 0118033