diff --git a/Demos/Demo33/ThSort.dfm b/Demos/Demo33/ThSort.dfm index 3b4c41e6..07127d4a 100644 --- a/Demos/Demo33/ThSort.dfm +++ b/Demos/Demo33/ThSort.dfm @@ -180,6 +180,7 @@ object ThreadSortForm: TThreadSortForm Engine = PythonEngine1 OnInitialization = SortModuleInitialization ModuleName = 'SortModule' + MultInterpretersSupport = mmiPerInterpreterGIL Errors = <> Left = 64 Top = 88 diff --git a/Install/MultiInstaller.exe b/Install/MultiInstaller.exe index 4908eecb..0e194ad4 100644 Binary files a/Install/MultiInstaller.exe and b/Install/MultiInstaller.exe differ diff --git a/Install/Setup.ini b/Install/Setup.ini index 8aac88b3..0676fc8f 100644 --- a/Install/Setup.ini +++ b/Install/Setup.ini @@ -53,6 +53,8 @@ LibSuffix=%s0 D27="Packages\Delphi\Delphi 10.4+\Python.dpk", "Packages\Delphi\Delphi 10.4+\dclPython.dpk" D28="Packages\Delphi\Delphi 10.4+\Python.dpk", "Packages\Delphi\Delphi 10.4+\dclPython.dpk" D29="Packages\Delphi\Delphi 10.4+\Python.dpk", "Packages\Delphi\Delphi 10.4+\dclPython.dpk" +D29="Packages\Delphi\Delphi 10.4+\Python.dpk", "Packages\Delphi\Delphi 10.4+\dclPython.dpk" +D37="Packages\Delphi\Delphi 10.4+\Python.dpk", "Packages\Delphi\Delphi 10.4+\dclPython.dpk" [Package - Python4DelphiVcl] Name=Python4Delphi Vcl @@ -62,6 +64,7 @@ LibSuffix=%s0 D27="Packages\Delphi\Delphi 10.4+\PythonVcl.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonVcl.dpk" D28="Packages\Delphi\Delphi 10.4+\PythonVcl.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonVcl.dpk" D29="Packages\Delphi\Delphi 10.4+\PythonVcl.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonVcl.dpk" +D37="Packages\Delphi\Delphi 10.4+\PythonVcl.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonVcl.dpk" [Package - Python4DelphiFmx] Name=Python4Delphi Fmx @@ -71,6 +74,7 @@ LibSuffix=%s0 D27="Packages\Delphi\Delphi 10.4+\PythonFmx.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonFmx.dpk" D28="Packages\Delphi\Delphi 10.4+\PythonFmx.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonFmx.dpk" D29="Packages\Delphi\Delphi 10.4+\PythonFmx.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonFmx.dpk" +D37="Packages\Delphi\Delphi 10.4+\PythonFmx.dpk", "Packages\Delphi\Delphi 10.4+\dclPythonFmx.dpk" ; Options format: ; [Options] diff --git a/Modules/DelphiVCL/TestVCL.py b/Modules/DelphiVCL/TestVCL.py index 2641d390..deae197c 100644 --- a/Modules/DelphiVCL/TestVCL.py +++ b/Modules/DelphiVCL/TestVCL.py @@ -40,6 +40,7 @@ def main(): f.Show() FreeConsole() Application.Run() + f.Free() main() diff --git a/Modules/DemoModule/InterpreterExecutor.py b/Modules/DemoModule/InterpreterExecutor.py new file mode 100644 index 00000000..97cca2ab --- /dev/null +++ b/Modules/DemoModule/InterpreterExecutor.py @@ -0,0 +1,35 @@ +#------------------------------------------------------------------------------- +# Name: InterpreterExecutor.py +# Purpose: Showcases the use of extension modules created with Delphi +# with the new in Python 3.14 InterpreterPoolExecutor +# You need python 3.14 to run this demo +# It uses the support module prime_utils which imports +# the delphi created extension module. +# Note that each interpreters has its own GIL and +# they are all running in parallel. +#------------------------------------------------------------------------------- + +from concurrent.futures import InterpreterPoolExecutor +from prime_utils import count_primes_in_range +import time + +def count_primes(max_num, num_interpreters=4): + chunk_size = max_num // num_interpreters + ranges = [(i, min(i + chunk_size - 1, max_num)) for i in range(2, max_num + 1, chunk_size)] + print(ranges) + + total = 0 + with InterpreterPoolExecutor(max_workers=num_interpreters) as executor: + results = executor.map(count_primes_in_range, ranges) + total = sum(results) + + return total + +if __name__ == "__main__": + max_number = 1_000_000 + start_time = time.time() + prime_count = count_primes(max_number) + end_time = time.time() + + print(f"Count of prime numbers up to {max_number}: {prime_count}") + print(f"Time taken: {end_time - start_time:.2f} seconds") \ No newline at end of file diff --git a/Modules/DemoModule/prime_utils.py b/Modules/DemoModule/prime_utils.py new file mode 100644 index 00000000..29abdea3 --- /dev/null +++ b/Modules/DemoModule/prime_utils.py @@ -0,0 +1,5 @@ +# prime_utils.py +from DemoModule import is_prime + +def count_primes_in_range(arange): + return sum(1 for n in range(arange[0], arange[1] + 1) if is_prime(n)) diff --git a/Modules/DemoModule/uMain.pas b/Modules/DemoModule/uMain.pas index 8df2b082..922ef62e 100644 --- a/Modules/DemoModule/uMain.pas +++ b/Modules/DemoModule/uMain.pas @@ -7,13 +7,14 @@ interface function PyInit_DemoModule: PPyObject; cdecl; implementation -Uses +uses + Winapi.Windows, System.Math, WrapDelphi; var - gEngine : TPythonEngine; - gModule : TPythonModule; + gEngine : TPythonEngine = nil; + gModule : TPythonModule = nil; function IsPrime(x: Integer): Boolean; // Naive implementation. It is just a demo @@ -46,6 +47,7 @@ function delphi_is_prime(self, args : PPyObject) : PPyObject; cdecl; function PyInit_DemoModule: PPyObject; begin + if not Assigned(gEngine) then try gEngine := TPythonEngine.Create(nil); gEngine.AutoFinalize := False; @@ -56,12 +58,18 @@ function PyInit_DemoModule: PPyObject; gModule.ModuleName := 'DemoModule'; gModule.AddMethod('is_prime', delphi_is_prime, 'is_prime(n) -> bool' ); + // We need to set this so that the module is not created by Initialzize + gModule.IsExtensionModule := True; + gModule.MultInterpretersSupport := mmiPerInterpreterGIL; + gEngine.LoadDllInExtensionModule; except + Exit(nil); end; - Result := gModule.Module; -end; + // The python import machinery will create the python module from ModuleDef + Result := gEngine.PyModuleDef_Init(@gModule.ModuleDef); +end; initialization finalization diff --git a/Modules/RttiModule/uMain.pas b/Modules/RttiModule/uMain.pas index 386b522f..38b5be84 100644 --- a/Modules/RttiModule/uMain.pas +++ b/Modules/RttiModule/uMain.pas @@ -17,6 +17,7 @@ implementation TDelphiFunctions = class public class function is_prime(const N: Integer): Boolean; static; + class procedure AfterModuleInit(Sender: TObject); end; var @@ -25,32 +26,37 @@ TDelphiFunctions = class gDelphiWrapper : TPyDelphiWrapper; DelphiFunctions: TDelphiFunctions; - - function PyInit_DemoModule: PPyObject; -var - Py : PPyObject; begin + if not Assigned(gEngine) then try gEngine := TPythonEngine.Create(nil); gEngine.AutoFinalize := False; gEngine.UseLastKnownVersion := True; + gDelphiWrapper := TPyDelphiWrapper.Create(nil); + gDelphiWrapper.Engine := gEngine; + + // !!It is important that the extension module is the last + // Engine client created gModule := TPythonModule.Create(nil); gModule.Engine := gEngine; gModule.ModuleName := 'DemoModule'; - gDelphiWrapper := TPyDelphiWrapper.Create(nil); - gDelphiWrapper.Engine := gEngine; + // Set IsExtensionModule so that the module is not created by Initialzize + gModule.IsExtensionModule := True; + gModule.MultInterpretersSupport := mmiPerInterpreterGIL; + gModule.OnAfterInitialization := TDelphiFunctions.AfterModuleInit; + gDelphiWrapper.Module := gModule; gEngine.LoadDllInExtensionModule; - Py := gDelphiWrapper.Wrap(DelphiFunctions, TObjectOwnership.soReference); - gModule.SetVar('delphi_funcs', Py); - gEngine.Py_DecRef(Py); except + Exit(nil); end; - Result := gModule.Module; + + // The python import machinery will create the python module from ModuleDef + Result := gEngine.PyModuleDef_Init(@gModule.ModuleDef); end; { TTestRttiAccess } @@ -58,6 +64,15 @@ function PyInit_DemoModule: PPyObject; { TDelphiFunctions } +class procedure TDelphiFunctions.AfterModuleInit(Sender: TObject); +var + Py : PPyObject; +begin + Py := gDelphiWrapper.Wrap(DelphiFunctions, TObjectOwnership.soReference); + gModule.SetVar('delphi_funcs', Py); + gEngine.Py_DecRef(Py); +end; + class function TDelphiFunctions.is_prime(const N: Integer): Boolean; // Naive implementation. It is just a demo... begin diff --git a/Source/Definition.Inc b/Source/Definition.Inc index efafdd5a..2aa3664f 100644 --- a/Source/Definition.Inc +++ b/Source/Definition.Inc @@ -175,6 +175,24 @@ {$DEFINE DELPHI11_OR_HIGHER} {$DEFINE DELPHI12_OR_HIGHER} {$ENDIF} +{$IFDEF VER370} // Delphi 13 + {$DEFINE DELPHI13} + {$DEFINE DELPHIXE2_OR_HIGHER} + {$DEFINE DELPHIXE3_OR_HIGHER} + {$DEFINE DELPHIXE4_OR_HIGHER} + {$DEFINE DELPHIXE5_OR_HIGHER} + {$DEFINE DELPHIXE6_OR_HIGHER} + {$DEFINE DELPHIXE7_OR_HIGHER} + {$DEFINE DELPHIXE8_OR_HIGHER} + {$DEFINE DELPHI10_OR_HIGHER} + {$DEFINE DELPHI10_1_OR_HIGHER} + {$DEFINE DELPHI10_2_OR_HIGHER} + {$DEFINE DELPHI10_3_OR_HIGHER} + {$DEFINE DELPHI10_4_OR_HIGHER} + {$DEFINE DELPHI11_OR_HIGHER} + {$DEFINE DELPHI12_OR_HIGHER} + {$DEFINE DELPHI13_OR_HIGHER} +{$ENDIF} ///////////////////////////////////////////////////////////////////////////// // Misc diff --git a/Source/MethodCallBack.pas b/Source/MethodCallBack.pas index 82b25a8e..47c782b6 100644 --- a/Source/MethodCallBack.pas +++ b/Source/MethodCallBack.pas @@ -137,7 +137,11 @@ function munmap(Addr: Pointer; Len: Integer): Integer; cdecl; PROT_WRITE =2; PROT_EXEC =4; MAP_PRIVATE =2; - MAP_ANON=$1000; + {$IFDEF MACOS} + MAP_ANON=$1000; + {$ELSE} + MAP_ANON=$20; + {$ENDIF MACOS} {$ENDIF} {$ENDIF} @@ -754,3 +758,5 @@ finalization FreeCallBacks; end. + + diff --git a/Source/PythonEngine.pas b/Source/PythonEngine.pas index c15fbf48..9e0343da 100644 --- a/Source/PythonEngine.pas +++ b/Source/PythonEngine.pas @@ -344,6 +344,20 @@ TPythonVersionProp = record PyBUF_READ = $100; PyBUF_WRITE = $200; +const + // constants used in PyModuleDef slots from moduleobject.h + Py_mod_create = 1; + Py_mod_exec = 2; + Py_mod_multiple_interpreters = 3; // Added in version 3.12 + Py_mod_gil = 4; // Added in version 3.13 + + Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED: Pointer = Pointer(0); + Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED: Pointer = Pointer(1); + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED: Pointer = Pointer(2); + + Py_MOD_GIL_USED: Pointer = Pointer(0); + Py_MOD_GIL_NOT_USED: Pointer = Pointer(1); + //####################################################### //## ## //## Non-Python specific constants ## @@ -628,6 +642,7 @@ TPythonVersionProp = record PyModuleDef_Slot = {$IFDEF CPUX86}packed{$ENDIF} record slot: integer; value: Pointer; + class function Make(slot: integer; value: Pointer): PyModuleDef_Slot; static; end; PPyModuleDef = ^PyModuleDef; @@ -643,6 +658,10 @@ TPythonVersionProp = record m_free : inquiry; end; + // signature of functions used in slots + Py_create_module_function = function(spec: PPyObject; def: PPyModuleDef):PPyObject; cdecl; + Py_exec_module_function = function(module: PPyObject): Integer; cdecl; + // pybuffer.h PPy_buffer = ^Py_Buffer; @@ -1029,7 +1048,7 @@ PyConfig = record // The followng needs updating when new versions are added const - ConfigOffests: TConfigOffsets = + ConfigOffsets: TConfigOffsets = {$IFDEF MSWINDOWS} {$IFDEF CPU64BITS} ((8, 80, 88, 144, 156, 160, 164, 172, 224, 104, 240, 248, 256, 272), @@ -1541,6 +1560,9 @@ TPythonInterface=class(TDynamicDll) PyCallable_Check: function(ob : PPyObject): integer; cdecl; PyModule_Create2: function(moduledef: PPyModuleDef; Api_Version: Integer):PPyObject; cdecl; + PyModuleDef_Init: function(moduledef: PPyModuleDef):PPyObject; cdecl; + PyModule_ExecDef: function(module: PPyObject; moduledef: PPyModuleDef):Integer; cdecl; + PyModule_FromDefAndSpec2: function(moduledef: PPyModuleDef; spec: PPyObject; Api_Version: Integer):PPyObject; cdecl; PyErr_BadArgument: function: integer; cdecl; PyErr_BadInternalCall: procedure; cdecl; PyErr_CheckSignals: function: integer; cdecl; @@ -1932,7 +1954,6 @@ TPythonInterface=class(TDynamicDll) function PyWeakref_CheckProxy( obj : PPyObject ) : Boolean; function PyBool_Check( obj : PPyObject ) : Boolean; function PyEnum_Check( obj : PPyObject ) : Boolean; - function Py_InitModule( const md : PyModuleDef) : PPyObject; // The following are defined as non-exported inline functions in object.h function Py_Type(ob: PPyObject): PPyTypeObject; inline; @@ -2117,6 +2138,7 @@ TPythonEngine = class(TPythonInterface) function ArrayToPyDict( const items : array of const) : PPyObject; function StringsToPyList( strings : TStrings ) : PPyObject; function StringsToPyTuple( strings : TStrings ) : PPyObject; + function Py_InitModule( const md : PyModuleDef) : PPyObject; procedure PyListToStrings(list: PPyObject; Strings: TStrings; ClearStrings: Boolean = True); procedure PyTupleToStrings( tuple: PPyObject; strings : TStrings ); function GetSequenceItem( sequence : PPyObject; idx : Integer ) : Variant; @@ -2173,7 +2195,7 @@ TPythonEngine = class(TPythonInterface) property IO: TPythonInputOutput read FIO write SetIO; property PyFlags: TPythonFlags read FPyFlags write SetPyFlags default DEFAULT_FLAGS; property RedirectIO: Boolean read FRedirectIO write FRedirectIO default True; - property UseWindowsConsole: Boolean read FUseWindowsConsole write FUseWindowsConsole default False; + property UseWindowsConsole: Boolean read FUseWindowsConsole write SetUseWindowsConsole default False; property OnAfterInit: TNotifyEvent read FOnAfterInit write FOnAfterInit; property OnSysPathInit: TSysPathInitEvent read FOnSysPathInit write FOnSysPathInit; property OnConfigInit: TConfigInitEvent read FOnConfigInit write FOnConfigInit; @@ -2282,7 +2304,6 @@ TMethodsContainer = class(TEngineClient) FMethodCount : Integer; FAllocatedMethodCount : Integer; FMethods : PPyMethodDef; - FModuleDef : PyModuleDef; FEventDefs: TEventDefs; procedure AllocMethods; @@ -2326,7 +2347,6 @@ TMethodsContainer = class(TEngineClient) property MethodCount : Integer read FMethodCount; property Methods[ idx : Integer ] : PPyMethodDef read GetMethods; property MethodsData : PPyMethodDef read FMethods; - property ModuleDef : PyModuleDef read FModuleDef; published property Events: TEventDefs read fEventDefs write fEventDefs stored StoreEventDefs; @@ -2481,65 +2501,71 @@ TErrors = class(TCollection) property Items[Index: Integer]: TError read GetError write SetError; default; end; + TMultIntperpretersSupport = (mmiSupported, mmiNotSupported, mmiPerInterpreterGIL); + {$IF not Defined(FPC) and (CompilerVersion >= 23)} [ComponentPlatformsAttribute(pidSupportedPlatforms)] {$IFEND} TPythonModule = class(TMethodsContainer) - protected - FModuleName : AnsiString; - FModule : PPyObject; - FClients : TList; - FErrors : TErrors; - FOnAfterInitialization : TNotifyEvent; - FDocString : TStringList; - - function GetClientCount : Integer; - function GetClients( idx : Integer ) : TEngineClient; - procedure SetErrors( val : TErrors ); - procedure SetModuleName( const val : AnsiString ); - procedure SetDocString( value : TStringList ); - public - // Constructors & destructors - constructor Create( AOwner : TComponent ); override; - destructor Destroy; override; + private + FModuleDef : PyModuleDef; + FMultInterpretersSupport: TMultIntperpretersSupport; + FEncodedDocString: AnsiString; + FIsExtensionModule: Boolean; + function Exec_Module(module: PPyObject): Integer; cdecl; // used in the slot + protected + FModuleName : AnsiString; + FModule : PPyObject; + FSlots: TArray; + FClients : TList; + FErrors : TErrors; + FDocString : TStringList; + FOnAfterInitialization : TNotifyEvent; + + function GetClientCount : Integer; + function GetClients( idx : Integer ) : TEngineClient; + procedure SetErrors( val : TErrors ); + procedure SetModuleName( const val : AnsiString ); + procedure SetDocString( value : TStringList ); + public + // Constructors & destructors + constructor Create( AOwner : TComponent ); override; + destructor Destroy; override; - // Public methods - procedure MakeModule; - procedure DefineDocString; - procedure Initialize; override; - procedure InitializeForNewInterpreter; - procedure AddClient(Client : TEngineClient); - procedure RemoveClient(Client : TEngineClient); - function ErrorByName( const AName : AnsiString ) : TError; - procedure RaiseError( const error, msg : AnsiString ); - procedure RaiseErrorFmt( const error, format : AnsiString; const Args : array of const ); - procedure RaiseErrorObj( const error, msg : AnsiString; obj : PPyObject ); - procedure BuildErrors; - procedure SetVar( const varName : AnsiString; value : PPyObject ); - function GetVar( const varName : AnsiString ) : PPyObject; - procedure DeleteVar( const varName : AnsiString ); - procedure ClearVars; - procedure SetVarFromVariant( const varName : AnsiString; const value : Variant ); - function GetVarAsVariant( const varName: AnsiString ) : Variant; + // Public methods + procedure MakeModuleDef; + procedure Initialize; override; + procedure InitializeForNewInterpreter; + procedure AddClient(Client : TEngineClient); + procedure RemoveClient(Client : TEngineClient); + function ErrorByName( const AName : AnsiString ) : TError; + procedure RaiseError( const error, msg : AnsiString ); + procedure RaiseErrorFmt( const error, format : AnsiString; const Args : array of const ); + procedure RaiseErrorObj( const error, msg : AnsiString; obj : PPyObject ); + procedure BuildErrors; + procedure SetVar( const varName : AnsiString; value : PPyObject ); + function GetVar( const varName : AnsiString ) : PPyObject; + procedure DeleteVar( const varName : AnsiString ); + procedure ClearVars; + procedure SetVarFromVariant( const varName : AnsiString; const value : Variant ); + function GetVarAsVariant( const varName: AnsiString ) : Variant; - // Public properties - property Module : PPyObject read FModule; - property Clients[ idx : Integer ] : TEngineClient read GetClients; - property ClientCount : Integer read GetClientCount; - published - property DocString : TStringList read FDocString write SetDocString; - property ModuleName : AnsiString read FModuleName write SetModuleName; - property Errors : TErrors read FErrors write SetErrors; - property OnAfterInitialization : TNotifyEvent read FOnAfterInitialization write FOnAfterInitialization; + // Public properties + property Module : PPyObject read FModule; + property ModuleDef : PyModuleDef read FModuleDef; + property IsExtensionModule: Boolean read FIsExtensionModule write FIsExtensionModule; + property Clients[ idx : Integer ] : TEngineClient read GetClients; + property ClientCount : Integer read GetClientCount; + published + property DocString : TStringList read FDocString write SetDocString; + property ModuleName : AnsiString read FModuleName write SetModuleName; + property MultInterpretersSupport: TMultIntperpretersSupport + read FMultInterpretersSupport write FMultInterpretersSupport; + property Errors : TErrors read FErrors write SetErrors; + property OnAfterInitialization : TNotifyEvent read FOnAfterInitialization write FOnAfterInitialization; end; -//------------------------------------------------------- -//-- -- -//--class: TPythonType derived from TGetSetContainer -- -//-- -- -//------------------------------------------------------- - { A B C +-------------------++------------------------------------------------------+ @@ -2556,15 +2582,15 @@ TPythonModule = class(TMethodsContainer) by GetSelf - a Python object must start at A. - - a Delphi class class must start at B + - a Delphi class must start at B - TPyObject.InstanceSize will return C-B - Sizeof(TPyObject) will return C-B - The total memory allocated for a TPyObject instance will be C-A, even if its InstanceSize is C-B. - - When turning a Python object pointer into a Delphi instance pointer, PythonToDelphi - will offset the pointer from A to B. - - When turning a Delphi instance into a Python object pointer, GetSelf will offset - Self from B to A. + - When turning a Python object pointer into a Delphi instance pointer, + PythonToDelphi will offset the pointer from A to B. + - When turning a Delphi instance into a Python object pointer, GetSelf + will offset Self from B to A. - Properties ob_refcnt and ob_type will call GetSelf to access their data. Further Notes: @@ -2581,7 +2607,6 @@ TPythonModule = class(TMethodsContainer) FreeInstance. - This class is heart of the P4D library. Pure magic!! } - // The base class of all new Python types TPyObject = class private function Get_ob_refcnt: NativeUInt; @@ -2751,8 +2776,15 @@ TTypeServices = class(TPersistent) property Mapping : TMappingServices read FMapping write FMapping; end; - // The component that initializes the Python type and - // that creates instances of itself. +//------------------------------------------------------- +//-- -- +//--class: TPythonType derived from TGetSetContainer -- +//-- -- +//------------------------------------------------------- + + // The component that initializes a Python type and + // creates instances of itself. + // The base class of all new Python types {$IF not Defined(FPC) and (CompilerVersion >= 23)} [ComponentPlatformsAttribute(pidSupportedPlatforms)] {$IFEND} @@ -3901,6 +3933,9 @@ procedure TPythonInterface.MapDll; PyDict_SetItemString := Import('PyDict_SetItemString'); PyDictProxy_New := Import('PyDictProxy_New'); PyModule_Create2 := Import('PyModule_Create2'); + PyModuleDef_Init := Import('PyModuleDef_Init'); + PyModule_ExecDef := Import('PyModule_ExecDef'); + PyModule_FromDefAndSpec2 := Import('PyModule_FromDefAndSpec2'); PyErr_Print := Import('PyErr_Print'); PyErr_SetNone := Import('PyErr_SetNone'); PyErr_SetObject := Import('PyErr_SetObject'); @@ -4425,21 +4460,6 @@ function TPythonInterface.PyObject_TypeCheck(obj: PPyObject; t: PPyTypeObject): Result := IsType(obj, t) or (PyType_IsSubtype(obj^.ob_type, t) = 1); end; -function TPythonInterface.Py_InitModule(const md: PyModuleDef): PPyObject; -Var - modules : PPyObject; -begin - CheckPython; - Result:= PyModule_Create2(@md, APIVersion); - if not Assigned(Result) then - GetPythonEngine.CheckError; - // To emulate Py_InitModule4 we need to add the module to sys.modules - modules := PyImport_GetModuleDict; - if PyDict_SetItemString(modules, md.m_name, Result) <> 0 then - GetPythonEngine.CheckError; -end; - - (*******************************************************) (** **) (** class TPythonTraceback **) @@ -4737,19 +4757,19 @@ procedure TPythonEngine.Initialize; procedure AssignPyFlags(var Config: PyConfig); begin - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.parser_debug])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.parser_debug])^ := IfThen(pfDebug in FPyFlags, 1, 0); - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.verbose])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.verbose])^ := IfThen(pfVerbose in FPyFlags, 1, 0); - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.interactive])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.interactive])^ := IfThen(pfInteractive in FPyFlags, 1, 0); - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.optimization_level])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.optimization_level])^ := IfThen(pfOptimize in FPyFlags, 1, 0); - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.site_import])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.site_import])^ := IfThen(pfNoSite in FPyFlags, 0, 1); - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.pathconfig_warnings])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.pathconfig_warnings])^ := IfThen(pfFrozen in FPyFlags, 1, 0); - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.use_environment])^ := + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.use_environment])^ := IfThen(pfIgnoreEnvironment in FPyFlags, 0, 1); end; @@ -4761,7 +4781,7 @@ procedure TPythonEngine.Initialize; begin // do not parse further - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.parse_argv])^ := 0; + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.parse_argv])^ := 0; for I := 0 to ParamCount do begin { @@ -4779,7 +4799,7 @@ procedure TPythonEngine.Initialize; Str := TempS; {$ENDIF} PyWideStringList_Append( - PPyWideStringList(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.argv]), + PPyWideStringList(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.argv]), PWCharT(Str)); end; end; @@ -4792,7 +4812,7 @@ procedure TPythonEngine.Initialize; begin if FPythonPath = '' then Exit; - PWSL := PPyWideStringList(PByte(@Config) + ConfigOffests[MinorVersion, + PWSL := PPyWideStringList(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.module_search_paths]); Paths := SplitString(string(FPythonPath), PathSep); for I := 0 to Length(Paths) - 1 do @@ -4803,7 +4823,7 @@ procedure TPythonEngine.Initialize; end; if PWSL^.length > 0 then - PInteger(PByte(@Config) + ConfigOffests[MinorVersion, + PInteger(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.module_search_paths_set])^ := 1; end; @@ -4822,16 +4842,16 @@ procedure TPythonEngine.Initialize; // Set programname and pythonhome if available if FProgramName <> '' then PyConfig_SetString(Config, - PPWcharT(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.program_name]), + PPWcharT(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.program_name]), PWCharT(StringToWCharTString(FProgramName))); if FPythonHome <> '' then PyConfig_SetString(Config, - PPWcharT(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.home]), + PPWcharT(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.home]), PWCharT(StringToWCharTString(FPythonHome))); // Set venv executable if available if FPythonExecutable <> '' then PyConfig_SetString(Config, - PPWcharT(PByte(@Config) + ConfigOffests[MinorVersion, TConfigFields.executable]), + PPWcharT(PByte(@Config) + ConfigOffsets[MinorVersion, TConfigFields.executable]), PWCharT(StringToWCharTString(FPythonExecutable))); // Set program arguments (sys.argv) @@ -5072,6 +5092,19 @@ procedure TPythonEngine.Initialize; if not Initialized then Initialize; + {$IFDEF MSWINDOWS} + // fix #504 + if not FRedirectIO and UseWindowsConsole then + PyRun_SimpleString( + 'import sys, io'#10 + + 'sys.stdout = io.TextIOWrapper(open("CONOUT$", "wb", buffering=0), ' + + 'encoding="utf-8", errors="replace", line_buffering=True)'#10 + + 'sys.stderr = io.TextIOWrapper(open("CONOUT$", "wb", buffering=0), ' + + 'encoding="utf-8", errors="replace", line_buffering=False)'#10 + + 'sys.stdin = io.TextIOWrapper(open("CONIN$", "rb", buffering=0), ' + + 'encoding="utf-8", errors="replace", line_buffering=True)'#10); + {$ENDIF} + if InitScript.Count > 0 then ExecStrings(InitScript); if Assigned(FOnAfterInit) then @@ -5127,10 +5160,12 @@ procedure TPythonEngine.InitWinConsole; FreeConsole; AllocConsole; SetConsoleTitle( 'Python console' ); + SetConsoleOutputCP(CP_UTF8); + SetConsoleCP(CP_UTF8); {$ENDIF} end; -procedure TPythonEngine.SetUseWindowsConsole( const Value : Boolean ); +procedure TPythonEngine.SetUseWindowsConsole(const Value: Boolean); begin FUseWindowsConsole := Value; if (csDesigning in ComponentState) then @@ -5570,9 +5605,7 @@ procedure TPythonEngine.RaiseError; s_type := GetTypeAsString(err_type); s_value := PyObjectAsString(err_value); - if (PyErr_GivenExceptionMatches(err_type, PyExc_SystemExit^) <> 0) then - raise Define( EPySystemExit.Create(''), s_type, s_value ) - else if (PyErr_GivenExceptionMatches(err_type, PyExc_StopIteration^) <> 0) then + if (PyErr_GivenExceptionMatches(err_type, PyExc_StopIteration^) <> 0) then raise Define( EPyStopIteration.Create(''), s_type, s_value ) else if (PyErr_GivenExceptionMatches(err_type, PyExc_KeyboardInterrupt^) <> 0) then raise Define( EPyKeyboardInterrupt.Create(''), s_type, s_value ) @@ -5800,13 +5833,13 @@ function TPythonEngine.TypeByName( const aTypeName : AnsiString ) : PPyTypeObjec raise Exception.CreateFmt(SCannotFindType, [aTypeName]); end; -function TPythonEngine.ModuleByName( const aModuleName : AnsiString ) : PPyObject; +function TPythonEngine.ModuleByName( const aModuleName : AnsiString ) : PPyObject; var i : Integer; begin for i := 0 to ClientCount - 1 do if Clients[i] is TPythonModule then - with TPythonModule( Clients[i] ) do + with TPythonModule(Clients[i]) do if ModuleName = aModuleName then begin Result := Module; @@ -6606,6 +6639,7 @@ procedure TPythonEngine.CheckError(ACatchStopEx : Boolean = False); var errtype, errvalue, errtraceback: PPyObject; SErrValue: string; + SystemExit: EPySystemExit; begin // PyErr_Fetch clears the error. The returned python objects are new references PyErr_Fetch(errtype, errvalue, errtraceback); @@ -6614,7 +6648,11 @@ procedure TPythonEngine.CheckError(ACatchStopEx : Boolean = False); Py_XDECREF(errtype); Py_XDECREF(errvalue); Py_XDECREF(errtraceback); - raise EPySystemExit.CreateResFmt(@SPyExcSystemError, [SErrValue]); + + SystemExit := EPySystemExit.CreateResFmt(@SPyExcSystemError, [SErrValue]); + SystemExit.EValue := SErrValue; + SystemExit.EName := 'SystemExit'; + raise SystemExit; end; var @@ -6706,6 +6744,64 @@ function TPythonEngine.PyUnicodeFromString(const AString: AnsiString): PPyObject end; +function TPythonEngine.Py_InitModule(const md: PyModuleDef): PPyObject; +// Implements multi-phase module intialization +var + modules, importlib, spec_func, module_name, args, spec: PPyObject; +begin + CheckPython; + + importlib := nil; + spec_func := nil; + module_name := nil; + args := nil; + spec := nil; + + try + // We need a spec and for that we need importlib; + importlib := PyImport_ImportModule('importlib.util'); + if not Assigned(importlib) then CheckError; + + // Get spec_from_loader function + spec_func := PyObject_GetAttrString(importlib, 'spec_from_loader'); + if not Assigned(spec_func) then CheckError; + + // Create module name + module_name := PyUnicode_FromString(md.m_name); + if not Assigned(module_name) then CheckError; + + // Create arguments tuple for spec_from_loader(name, loader) + args := MakePyTuple([module_name, Py_None]); + + // Create the module specification + spec := PyObject_CallObject(spec_func, args); + if not Assigned(spec) then CheckError; + + // Create the module from the definition and spec + Result := PyModule_FromDefAndSpec2(@md, spec, APIVersion); + if not Assigned(spec) then CheckError; + + // Execute the module (triggers Py_mod_exec slot) + if (PyModule_ExecDef(Result, @md) < 0) then + begin + Py_DECREF(Result); + CheckError; + end; + + finally + Py_XDECREF(importlib); + Py_XDECREF(spec_func); + Py_XDECREF(module_name); + Py_XDECREF(args); + Py_XDECREF(spec); + end; + + // We need to add the module to sys.modules + modules := PyImport_GetModuleDict; + if PyDict_SetItemString(modules, md.m_name, Result) <> 0 then + GetPythonEngine.CheckError; +end; + (*******************************************************) (** **) (** class TEngineClient **) @@ -7568,6 +7664,7 @@ constructor TPythonModule.Create( AOwner : TComponent ); FClients := TList.Create; FErrors := TErrors.Create(Self); FDocString := TStringList.Create; + FDocString.TrailingLineBreak := False; end; destructor TPythonModule.Destroy; @@ -7583,52 +7680,59 @@ procedure TPythonModule.SetDocString( value : TStringList ); FDocString.Assign( value ); end; -procedure TPythonModule.DefineDocString; +procedure TPythonModule.MakeModuleDef; var - doc : PPyObject; + P: Pointer; begin - with Engine do - begin - if DocString.Text <> '' then - begin - doc := - PyUnicodeFromString(CleanString(FDocString.Text, False)); - PyObject_SetAttrString( FModule, '__doc__', doc ); - Py_XDecRef(doc); - CheckError(False); - end; - end; -end; + FillChar(FModuleDef, SizeOf(FModuleDef), 0); + FModuleDef.m_base.ob_refcnt := 1; + FModuleDef.m_name := PAnsiChar(ModuleName); + FModuleDef.m_methods := MethodsData; + FModuleDef.m_size := 0; -procedure TPythonModule.MakeModule; -begin - CheckEngine; - if Assigned(FModule) then - Exit; - with Engine do - begin - FillChar(FModuleDef, SizeOf(FModuleDef), 0); - FModuleDef.m_base.ob_refcnt := 1; - FModuleDef.m_name := PAnsiChar(ModuleName); - FModuleDef.m_methods := MethodsData; - FModuleDef.m_size := -1; - FModule := Py_InitModule( ModuleDef ); - DefineDocString; + // Doc string + if FDocString.Count > 0 then + begin + FEncodedDocString := UTF8Encode(CleanString(FDocString.Text, False)); + FModuleDef.m_doc := PAnsiChar(FEncodedDocString); + end; + + // Fill the m_slots for multi-phase initialization + FSlots := [PyModuleDef_Slot.Make(Py_mod_exec, + GetCallBack(Self, @TPythonModule.Exec_Module, 1, DEFAULT_CALLBACK_TYPE))]; + + if (Engine.MajorVersion > 3) or (Engine.MinorVersion >= 12) then + begin + case FMultInterpretersSupport of + mmiNotSupported: P := Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED; + mmiPerInterpreterGIL: P := Py_MOD_PER_INTERPRETER_GIL_SUPPORTED; + else + P := Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; end; + FSlots := FSlots + [PyModuleDef_Slot.Make(Py_mod_multiple_interpreters, P)]; + end; + FSlots := FSlots + [PyModuleDef_Slot.Make(0, nil)]; + + FModuleDef.m_slots := @FSlots[0]; end; procedure TPythonModule.Initialize; -var - i : Integer; begin inherited; - FModule := nil; - MakeModule; - for i := 0 to ClientCount - 1 do - Clients[i].ModuleReady(Self); - BuildErrors; - if Assigned(FOnAfterInitialization) then - FOnAfterInitialization( Self ); + + if Assigned(FModule) then Exit; + + MakeModuleDef; + // Py_InitModule will call Exec_Module which will + // - Set FModule + // - initialize clients + // - Call OnInitialized + CheckEngine; + + // Extension modules are intilized directly from ModuleDef + if FIsExtensionModule then Exit; + + FModule := Engine.Py_InitModule(FModuleDef); end; procedure TPythonModule.InitializeForNewInterpreter; @@ -7667,6 +7771,21 @@ function TPythonModule.ErrorByName( const AName : AnsiString ) : TError; raise Exception.CreateFmt(SCouldNotFindError, [AName] ); end; +function TPythonModule.Exec_Module(module: PPyObject): Integer; +// Executed via the m_slots of PyModuleDef as part of the +// multi-phase module initialization +var + I : Integer; +begin + FModule := module; + for I := 0 to ClientCount - 1 do + Clients[I].ModuleReady(Self); + BuildErrors; + if Assigned(FOnAfterInitialization) then + FOnAfterInitialization(Self); + Result := 0; +end; + procedure TPythonModule.RaiseError( const error, msg : AnsiString ); begin ErrorByName( error ).RaiseError( msg ); @@ -7696,7 +7815,7 @@ procedure TPythonModule.BuildErrors; CheckEngine; with Engine do begin - d := PyModule_GetDict( Module ); + d := PyModule_GetDict(FModule); if not Assigned(d) then Exit; for i := 0 to Errors.Count - 1 do @@ -7715,7 +7834,7 @@ procedure TPythonModule.SetVar( const varName : AnsiString; value : PPyObject ); begin if Assigned(FEngine) and Assigned( FModule ) then begin - if Engine.PyObject_SetAttrString(Module, PAnsiChar(varName), value ) <> 0 then + if Engine.PyObject_SetAttrString(FModule, PAnsiChar(varName), value ) <> 0 then raise EPythonError.CreateFmt(SCouldNotSetVar, [varName, ModuleName]); end else @@ -7729,21 +7848,21 @@ function TPythonModule.GetVar( const varName : AnsiString ) : PPyObject; begin if Assigned(FEngine) and Assigned( FModule ) then begin - Result := Engine.PyObject_GetAttrString(Module, PAnsiChar(varName) ); + Result := Engine.PyObject_GetAttrString(FModule, PAnsiChar(varName) ); Engine.PyErr_Clear; end else raise EPythonError.CreateFmt(SCannotSetVarNoInit, [varName, ModuleName]); end; -procedure TPythonModule.DeleteVar( const varName : AnsiString ); +procedure TPythonModule.DeleteVar(const varName : AnsiString); var dict : PPyObject; begin if Assigned(FEngine) and Assigned( FModule ) then with Engine do begin - dict := PyModule_GetDict( Module ); + dict := PyModule_GetDict(FModule); if not Assigned(dict) then raise EPythonError.CreateFmt(SCannotGetDict, [ModuleName] ); PyDict_DelItemString( dict, PAnsiChar(varName) ); @@ -7758,7 +7877,7 @@ procedure TPythonModule.ClearVars; begin if Assigned(FEngine) and Assigned( FModule ) then with Engine do begin - dict := PyModule_GetDict( Module ); + dict := PyModule_GetDict(FModule); PyDict_Clear(dict); end; end; @@ -8890,7 +9009,7 @@ procedure TPythonType.InitServices; begin tp_init := TPythonType_InitSubtype; tp_alloc := FEngine.PyType_GenericAlloc; - tp_new := GetCallBack( Self, @TPythonType.NewSubtypeInst, 3, DEFAULT_CALLBACK_TYPE); + tp_new := GetCallBack(Self, @TPythonType.NewSubtypeInst, 3, DEFAULT_CALLBACK_TYPE); tp_free := FEngine.PyObject_Free; tp_methods := MethodsData; tp_members := MembersData; @@ -9532,7 +9651,8 @@ procedure TPythonThread.Execute; finally PyGILState_Release(gilstate); end; - end else + end + else begin gilstate := PyGILState_Ensure(); global_state := PyThreadState_Get; @@ -9545,16 +9665,27 @@ procedure TPythonThread.Execute; ((FMajorVersion = 3) and (FMinorVersion < 12)) or PyStatus_Exception(Py_NewInterpreterFromConfig(@fThreadState, @Config)) then - fThreadState := Py_NewInterpreter; + fThreadState := Py_NewInterpreter + else if Assigned(IOPythonModule) then + // flag IOPythonModule as per interpreter GIL compatible + TPythonModule(IOPythonModule).MultInterpretersSupport := mmiPerInterpreterGIL; - if Assigned( fThreadState) then + if Assigned(fThreadState) then begin PyThreadState_Swap(fThreadState); + // Redirect IO + if RedirectIO and Assigned(IO) and Assigned(IOPythonModule) then + begin + TPythonModule(IOPythonModule).InitializeForNewInterpreter; + DoRedirectIO; + end; + // Execute the python code ExecuteWithPython; Py_EndInterpreter( fThreadState); PyThreadState_Swap(global_state); PyGILState_Release(gilstate); - end else + end + else raise EPythonError.Create(SCannotCreateThreadState); end; end; @@ -10131,5 +10262,14 @@ procedure ThreadPythonExec(ExecuteProc : TProc; TerminateProc : TProc; {$ENDIF FPC} +{ PyModuleDef_Slot } + +class function PyModuleDef_Slot.Make(slot: integer; + value: Pointer): PyModuleDef_Slot; +begin + Result.slot := slot; + Result.value := value; +end; + end. diff --git a/Source/VarPyth.pas b/Source/VarPyth.pas index 52607cf0..9e2c8816 100644 --- a/Source/VarPyth.pas +++ b/Source/VarPyth.pas @@ -71,7 +71,8 @@ function VarIsSubtypeOf(const ADerived, AType : Variant): Boolean; function VarIsNone(const AValue : Variant): Boolean; function VarIsTrue(const AValue : Variant): Boolean; -function VarModuleHasObject(const AModule : Variant; aObj: AnsiString): Boolean; +function VarModuleHasObject(const AModule : Variant; const aObj: AnsiString): Boolean; +function VarHasAttr(const AValue: Variant; const AAttr: AnsiString): Boolean; function NewPythonList( const ASize : Integer = 0 ): Variant; function NewPythonTuple( const ASize : Integer ): Variant; @@ -570,15 +571,22 @@ function VarIsTrue(const AValue : Variant): Boolean; Result := AValue; // the cast into a boolean will call the PyObject_IsTrue API. end; -function VarModuleHasObject(const AModule : Variant; aObj: AnsiString): Boolean; +function VarModuleHasObject(const AModule : Variant; const aObj: AnsiString): + Boolean; begin with GetPythonEngine do - Result := VarIsPython(AModule) and - PyModule_Check(ExtractPythonObjectFrom(AModule)) and + Result := VarIsPythonModule(AModule) and Assigned(PyDict_GetItemString( PyModule_GetDict(ExtractPythonObjectFrom(AModule)),PAnsiChar(aObj))); end; +function VarHasAttr(const AValue: Variant; const AAttr: AnsiString): Boolean; +begin + with GetPythonEngine do + Result := VarIsPython(AValue) and + (PyObject_HasAttrString(ExtractPythonObjectFrom(AValue), PAnsiChar(AAttr)) = 1); +end; + function NewPythonList( const ASize : Integer = 0 ): Variant; var _list : PPyObject; diff --git a/Source/WrapDelphi.pas b/Source/WrapDelphi.pas index a19bc718..e399dbf4 100644 --- a/Source/WrapDelphi.pas +++ b/Source/WrapDelphi.pas @@ -1003,7 +1003,7 @@ TPyDelphiWrapper = class(TEngineClient, IFreeNotificationSubscriber) implementation -Uses +uses Math, StrUtils, RTLConsts, @@ -2636,11 +2636,13 @@ procedure SetPropValue(Instance: TObject; PropInfo: PPropInfo; const Value: Vari end; {$ENDIF} +{$HINTS OFF} function Abort_Wrapper(pself, args: PPyObject): PPyObject; cdecl; begin Result := nil; Abort; end; +{$HINTS ON} Type // Used for class registration by TPyDelphiWrapper fClassRegister @@ -5419,12 +5421,11 @@ procedure TPyDelphiWrapper.Initialize; with TPythonType(fHelperClassRegister.Objects[i]) do if not Initialized then Initialize; // Initialize module - if Assigned(FModule) then begin + if Assigned(FModule) then + begin + CreateModuleFunctions; if Module.Initialized then - begin - CreateModuleFunctions; - CreateModuleVars; - end + CreateModuleVars else Module.AddClient( Self ); end; @@ -5433,7 +5434,6 @@ procedure TPyDelphiWrapper.Initialize; procedure TPyDelphiWrapper.ModuleReady(Sender : TObject); begin inherited; - CreateModuleFunctions; CreateModuleVars; end; @@ -5542,13 +5542,13 @@ procedure TPyDelphiWrapper.SetModule(const Value: TPythonModule); TPythonType(fHelperClassRegister.Objects[i]).Module := Value; if Assigned(FModule) then if Initialized and (ComponentState * [csDesigning, csLoading] = []) then + begin + CreateModuleFunctions; if FModule.Initialized then - begin - CreateModuleFunctions; - CreateModuleVars; - end + CreateModuleVars else FModule.AddClient(Self); + end; end; end;