From fa81450a890be39ba64838605a4bb7be31fcbe9c Mon Sep 17 00:00:00 2001 From: AlexNg Date: Thu, 14 Dec 2023 22:58:53 +0800 Subject: [PATCH 01/17] + Added docstring & exported types --- src/thread/__init__.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/thread/__init__.py b/src/thread/__init__.py index c59831c..13a87d3 100644 --- a/src/thread/__init__.py +++ b/src/thread/__init__.py @@ -1,10 +1,34 @@ +""" +## Thread Library +Documentation at https://thread.ngjx.org + + +--- + +Released under the GPG-3 License + +Copyright (c) thread.ngjx.org, All rights reserved +""" + +""" +This file contains the exports to +```py +import thread +``` +""" + + +# Export Core from .thread import ( Thread, - ParallelProcessing, + ParallelProcessing ) + from . import ( + _types as types, exceptions ) +# Configuration from .utils import Settings From 38cdb397834aa11600b2a9a61c47e3fcc7229ee9 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Thu, 14 Dec 2023 22:59:37 +0800 Subject: [PATCH 02/17] Migrated types threads -> _types --- src/thread/_types.py | 21 +++++++++++++++++++++ src/thread/thread.py | 29 ++++++++++++----------------- 2 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 src/thread/_types.py diff --git a/src/thread/_types.py b/src/thread/_types.py new file mode 100644 index 0000000..b376a2c --- /dev/null +++ b/src/thread/_types.py @@ -0,0 +1,21 @@ +""" +## Types + +Documentation: https://thread.ngjx.org +""" + +from typing import Any, Literal + +ThreadStatus = Literal[ + 'Idle', + 'Running', + 'Invoking hooks', + 'Completed', + + 'Errored', + 'Kill Scheduled', + 'Killed' +] +Data_In = Any +Data_Out = Any +Overflow_In = Any diff --git a/src/thread/thread.py b/src/thread/thread.py index 6c0512e..b1fa753 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -1,35 +1,30 @@ +""" +## Core of thread + +```py +Thread() +ParallelProcessing() +``` +""" + import sys import time import signal import threading +from functools import wraps from . import exceptions from .utils.config import Settings from .utils.algorithm import chunk_split -from functools import wraps +from ._types import ThreadStatus, Data_In, Data_Out, Overflow_In from typing import ( Any, List, - Callable, Union, Optional, Literal, + Callable, Union, Optional, Mapping, Sequence, Tuple ) -ThreadStatus = Literal[ - 'Idle', - 'Running', - 'Invoking hooks', - 'Completed', - - 'Errored', - 'Kill Scheduled', - 'Killed' -] -Data_In = Any -Data_Out = Any -Overflow_In = Any - - Threads: set['Thread'] = set() class Thread(threading.Thread): """ From 301224d3849c4a196226097bda2197c2a09c7a16 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Thu, 14 Dec 2023 23:00:15 +0800 Subject: [PATCH 03/17] + Added docstring --- src/thread/exceptions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index bdc6048..1f039bf 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -1,3 +1,7 @@ +""" +## Thread Exceptions +""" + import traceback from typing import Any, Optional, Sequence, Tuple From 2381f10e14e17df22a69679c0643745b83ab41b9 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Thu, 14 Dec 2023 23:03:57 +0800 Subject: [PATCH 04/17] + Added placeholder documentation link --- src/thread/exceptions.py | 2 ++ src/thread/thread.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index 1f039bf..91b057c 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -1,5 +1,7 @@ """ ## Thread Exceptions + +Documentation: https://thread.ngjx.org """ import traceback diff --git a/src/thread/thread.py b/src/thread/thread.py index b1fa753..9bae2b7 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -2,9 +2,11 @@ ## Core of thread ```py -Thread() -ParallelProcessing() +class Thread: ... +class ParallelProcessing: ... ``` + +Documentation: https://thread.ngjx.org """ import sys From 316184d57cb176a28a13c019a72294dcc1e0c47d Mon Sep 17 00:00:00 2001 From: AlexNg Date: Thu, 14 Dec 2023 23:33:20 +0800 Subject: [PATCH 05/17] Exported HookFunction typing to _types --- src/thread/_types.py | 4 +++- src/thread/thread.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/thread/_types.py b/src/thread/_types.py index b376a2c..7c7ee00 100644 --- a/src/thread/_types.py +++ b/src/thread/_types.py @@ -4,7 +4,7 @@ Documentation: https://thread.ngjx.org """ -from typing import Any, Literal +from typing import Any, Literal, Callable, Union ThreadStatus = Literal[ 'Idle', @@ -19,3 +19,5 @@ Data_In = Any Data_Out = Any Overflow_In = Any + +HookFunction = Callable[[Data_Out], Union[Any, None]] diff --git a/src/thread/thread.py b/src/thread/thread.py index 9bae2b7..00131ab 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -19,7 +19,7 @@ class ParallelProcessing: ... from .utils.config import Settings from .utils.algorithm import chunk_split -from ._types import ThreadStatus, Data_In, Data_Out, Overflow_In +from ._types import ThreadStatus, Data_In, Data_Out, Overflow_In, HookFunction from typing import ( Any, List, Callable, Union, Optional, @@ -37,7 +37,7 @@ class Thread(threading.Thread): """ status : ThreadStatus - hooks : List[Callable[[Data_Out], Union[Any, None]]] + hooks : List[HookFunction] returned_value: Data_Out errors : List[Exception] @@ -208,7 +208,7 @@ def is_alive(self) -> bool: return super().is_alive() - def add_hook(self, hook: Callable[[Data_Out], Union[Any, None]]) -> None: + def add_hook(self, hook: HookFunction) -> None: """ Adds a hook to the thread ------------------------- From de98eb9284fce07e1b74e022980e1028e756e3dc Mon Sep 17 00:00:00 2001 From: AlexNg Date: Thu, 14 Dec 2023 23:38:42 +0800 Subject: [PATCH 06/17] Exported TargetFunction typing to _types --- src/thread/_types.py | 14 +++++++++++--- src/thread/thread.py | 12 ++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/thread/_types.py b/src/thread/_types.py index 7c7ee00..9204c47 100644 --- a/src/thread/_types.py +++ b/src/thread/_types.py @@ -6,6 +6,14 @@ from typing import Any, Literal, Callable, Union + +# Descriptive Types +Data_In = Any +Data_Out = Any +Overflow_In = Any + + +# Variable Types ThreadStatus = Literal[ 'Idle', 'Running', @@ -16,8 +24,8 @@ 'Kill Scheduled', 'Killed' ] -Data_In = Any -Data_Out = Any -Overflow_In = Any + +# Function types HookFunction = Callable[[Data_Out], Union[Any, None]] +TargetFunction = Callable[..., Data_Out] diff --git a/src/thread/thread.py b/src/thread/thread.py index 00131ab..92b42a2 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -19,10 +19,10 @@ class ParallelProcessing: ... from .utils.config import Settings from .utils.algorithm import chunk_split -from ._types import ThreadStatus, Data_In, Data_Out, Overflow_In, HookFunction +from ._types import ThreadStatus, Data_In, Data_Out, Overflow_In, TargetFunction, HookFunction from typing import ( Any, List, - Callable, Union, Optional, + Callable, Optional, Mapping, Sequence, Tuple ) @@ -51,7 +51,7 @@ class Thread(threading.Thread): def __init__( self, - target: Callable[..., Data_Out], + target: TargetFunction, args: Sequence[Data_In] = (), kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), @@ -100,7 +100,7 @@ def __init__( ) - def _wrap_target(self, target: Callable[..., Data_Out]) -> Callable[..., Data_Out]: + def _wrap_target(self, target: TargetFunction) -> TargetFunction: """Wraps the target function""" @wraps(target) def wrapper(*args: Any, **kwargs: Any) -> Any: @@ -344,7 +344,7 @@ class ParallelProcessing: def __init__( self, - function: Callable[..., Data_Out], + function: TargetFunction, dataset: Sequence[Data_In], max_threads: int = 8, @@ -385,7 +385,7 @@ def __init__( def _wrap_function( self, - function: Callable[..., Data_Out] + function: TargetFunction ) -> Callable[..., List[Data_Out]]: @wraps(function) def wrapper(index: int, data_chunk: Sequence[Data_In], *args: Any, **kwargs: Any) -> List[Data_Out]: From cf9dfbb8063d42746fae3265f9e8cc55aa927d91 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Fri, 15 Dec 2023 16:32:06 +0800 Subject: [PATCH 07/17] + Decorator base class --- src/thread/decorators.py | 146 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 src/thread/decorators.py diff --git a/src/thread/decorators.py b/src/thread/decorators.py new file mode 100644 index 0000000..8314e02 --- /dev/null +++ b/src/thread/decorators.py @@ -0,0 +1,146 @@ +""" +## Decorators + +Documentation: https://thread.ngjx.org +""" + +from functools import wraps +from abc import ABC, abstractmethod + +from ._types import TargetFunction, Overflow_In, Data_Out, Data_In +from typing import Any, Callable, Mapping, Tuple + +from .thread import Thread +from .exceptions import AbstractInvokationError, ArgumentValidationError + + +class DecoratorBase(ABC): + """ + Decorator Base Class + + This should only be used from inheriting classes + + Use Case + -------- + ```py + class MyDecorator(DecoratorBase): + # DocString here for decorator without arguments + + # Arguments to the decorator are here + args: Tuple[Any, ...] + kwargs: Mapping[str, Any] + + def __init__(self, *args, **kwargs): + # DocString here for decorator with arguments + # If you want to have type hinting + # If you want to validate arguments to decorator + super().__init__(*args, **kwargs) + + def __validate__(self, args, kwargs): + # If you want to validate arguments to the wrapped function + return bool + + def __execute__(self, func, args, kwargs): + return func(*args, **kwargs) + ``` + """ + + args: Tuple[Any, ...] + kwargs: Mapping[str, Any] + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """init""" + self.args = args + self.kwargs = kwargs + + def __call__(self, *args: Any, **kwargs: Any) -> Data_Out: + """ + The main logic to allow decorators to be invoked: + + ```py + @MyDecorator + def a(): ... + + @MyDecorator() + def b(): ... + + @MyDecorator(1, 2, 3) + def c(): ... + ``` + """ + if callable(self.args[0]) and self.__validate__(args, kwargs): + func = self.args[0] + return self.__execute__(func, args, kwargs) + + elif not callable(self.args[0]): + func = args[0] + + @wraps(self.__execute__) + def wrapper(*args, **kwargs): + if self.__validate__(args, kwargs): + return self.__execute__(func, args, kwargs) + return wrapper + + else: + raise ArgumentValidationError() + + + @abstractmethod + def __execute__(self, func: Callable[..., Data_Out], args: Tuple[Any, ...], kwargs: Mapping[str, Any]) -> Data_Out: + """ + The main code returning the result of the wrapped method + + Parameters + ---------- + :param func: The wrapped function + :param args: The tuple of parsed arguments + :param kwargs: The tuple of parsed keyword arguments + + Returns + ------- + :returns Data_Out: The result of the wrapped function + + Raises + ------ + AbstractInvocationsError: When the base class abstract method is invoked + + Use Case + -------- + ```py + class MyDecorator(DecoratorBase): + ... + def __execute__(self, func, args, kwargs): + return func(*args, **kwargs) + ``` + """ + raise AbstractInvokationError('__execute__') + + @abstractmethod + def __validate__(self, args: Tuple[Any, ...], kwargs: Mapping[str, Any]) -> bool: + """ + Validate arguments parsed to the wrapped method + + Parameters + ---------- + :param args: Tuple of parsed positional arguments + :param kwargs: Dictionary of parsed keyword arguments + + Returns + ------- + :returns bool: True if the arguments are valid + + Raises + ------ + AbstractInvokationError: When the base class abstract method is invoked + + Use Case + -------- + ```py + @MyDecorator + def myfunction(*args, **kwargs): ... + + myfunction(1, 2, 3) # Arguments parsed here is validated here + ``` + """ + raise AbstractInvokationError('__validate__') + From cf0cbee0fde43332ebc96af5b0ca6652d4e0d7f5 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Fri, 15 Dec 2023 16:32:32 +0800 Subject: [PATCH 08/17] + Decorator exceptions --- src/thread/exceptions.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index 91b057c..266e78a 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -8,7 +8,7 @@ from typing import Any, Optional, Sequence, Tuple -class ThreadErrorBase(Exception): +class ErrorBase(Exception): """Base exception class for all errors within this library""" message: str = 'Something went wrong!' def __init__(self, message: Optional[str] = None, *args: Any, **kwargs: Any) -> None: @@ -16,19 +16,21 @@ def __init__(self, message: Optional[str] = None, *args: Any, **kwargs: Any) -> super().__init__(message, *args, **kwargs) -class ThreadStillRunningError(ThreadErrorBase): + +# THREAD ERRORS # +class ThreadStillRunningError(ErrorBase): """Exception class for attempting to invoke a method which requries the thread not be running, but isn't""" message: str = 'Thread is still running, unable to invoke method. You can wait for the thread to terminate with `Thread.join()` or check with `Thread.is_alive()`' -class ThreadNotRunningError(ThreadErrorBase): +class ThreadNotRunningError(ErrorBase): """Exception class for attempting to invoke a method which requires the thread to be running, but isn't""" message: str = 'Thread is not running, unable to invoke method. Have you ran `Thread.start()`?' -class ThreadNotInitializedError(ThreadErrorBase): +class ThreadNotInitializedError(ErrorBase): """Exception class for attempting to invoke a method which requires the thread to be initialized, but isn't""" message: str = 'Thread is not initialized, unable to invoke method.' -class HookRuntimeError(ThreadErrorBase): +class HookRuntimeError(ErrorBase): """Exception class for hook runtime errors""" message: str = 'Encountered runtime errors in hooks' count: int = 0 @@ -52,11 +54,15 @@ def __init__(self, message: Optional[str] = '', extra: Sequence[Tuple[Exception, super().__init__(new_message) - # Python 3.9 doesn't support Exception.add_note() - # def add_exception_case(self, func_name: str, error: Exception): - # self.count += 1 - # trace = '\n'.join(traceback.format_stack()) - # self.add_note(f'\n{self.count}. {func_name}\n>>>>>>>>>>') - # self.add_note(f'{trace}\n{error}') - # self.add_note('<<<<<<<<<<') +# DECORATOR ERRORS # +class AbstractInvokationError(ErrorBase): + """Exception class for attempting to invoke an abstract method which is only accessible from inheriting classes""" + message: str = 'Attempt to invoke abstract method [{method_name}]!' + + def __init__(self, method_name: str) -> None: + super().__init__(self.message.format(method_name = method_name)) + +class ArgumentValidationError(ErrorBase): + """Exception class for when validating arguments passed to the wrapped method fails""" + message: str = 'Validation for arguments passed to the wrapped method failed' From 4f97cabd2b9df5189abe683bcaf64af12920d722 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Fri, 15 Dec 2023 16:41:40 +0800 Subject: [PATCH 09/17] + Exported threaded decorator --- src/thread/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/thread/__init__.py b/src/thread/__init__.py index 13a87d3..560f576 100644 --- a/src/thread/__init__.py +++ b/src/thread/__init__.py @@ -30,5 +30,12 @@ exceptions ) + +# Export decorators +from .decorators import ( + threaded +) + + # Configuration from .utils import Settings From ddf8efad26ea7e95a63d701469e4a68086f975df Mon Sep 17 00:00:00 2001 From: AlexNg Date: Fri, 15 Dec 2023 18:39:20 +0800 Subject: [PATCH 10/17] + Added optional additional info to valaidation error --- src/thread/exceptions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index 266e78a..b807769 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -66,3 +66,6 @@ def __init__(self, method_name: str) -> None: class ArgumentValidationError(ErrorBase): """Exception class for when validating arguments passed to the wrapped method fails""" message: str = 'Validation for arguments passed to the wrapped method failed' + + def __init__(self, additional: Optional[str] = '') -> None: + super().__init__(self.message + f'\n{additional}') From f557f7eae54f8a95f30f21917d6e71c6972134b4 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Fri, 15 Dec 2023 18:39:43 +0800 Subject: [PATCH 11/17] + Added thread.threaded decorator --- src/thread/decorators.py | 117 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 6 deletions(-) diff --git a/src/thread/decorators.py b/src/thread/decorators.py index 8314e02..17c1b3b 100644 --- a/src/thread/decorators.py +++ b/src/thread/decorators.py @@ -4,11 +4,12 @@ Documentation: https://thread.ngjx.org """ +import inspect from functools import wraps from abc import ABC, abstractmethod from ._types import TargetFunction, Overflow_In, Data_Out, Data_In -from typing import Any, Callable, Mapping, Tuple +from typing import Any, Callable, Mapping, Tuple, Sequence, Optional, Union from .thread import Thread from .exceptions import AbstractInvokationError, ArgumentValidationError @@ -36,6 +37,9 @@ def __init__(self, *args, **kwargs): # If you want to validate arguments to decorator super().__init__(*args, **kwargs) + def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs) + def __validate__(self, args, kwargs): # If you want to validate arguments to the wrapped function return bool @@ -49,11 +53,20 @@ def __execute__(self, func, args, kwargs): kwargs: Mapping[str, Any] def __init__(self, *args: Any, **kwargs: Any) -> None: - """init""" + """ + Decorator Arguments + + There arguments will `NOT` be parsed to the wrapped function + + Parameters + ---------- + :param *: Positional arguments + :param **: Keyword arguments + """ self.args = args self.kwargs = kwargs - def __call__(self, *args: Any, **kwargs: Any) -> Data_Out: + def __call__(self, *args: Any, **kwargs: Any) -> Union[Any, Callable[..., Any]]: """ The main logic to allow decorators to be invoked: @@ -68,17 +81,20 @@ def b(): ... def c(): ... ``` """ - if callable(self.args[0]) and self.__validate__(args, kwargs): + is_callable = (len(self.args) >= 1) and callable(self.args[0]) + if is_callable and self.__validate__(args, kwargs): func = self.args[0] return self.__execute__(func, args, kwargs) - elif not callable(self.args[0]): + elif not is_callable: func = args[0] @wraps(self.__execute__) def wrapper(*args, **kwargs): if self.__validate__(args, kwargs): return self.__execute__(func, args, kwargs) + else: + raise ArgumentValidationError() return wrapper else: @@ -86,7 +102,7 @@ def wrapper(*args, **kwargs): @abstractmethod - def __execute__(self, func: Callable[..., Data_Out], args: Tuple[Any, ...], kwargs: Mapping[str, Any]) -> Data_Out: + def __execute__(self, func: Callable[..., Data_Out], args: Tuple[Any, ...], kwargs: Mapping[str, Any]): """ The main code returning the result of the wrapped method @@ -144,3 +160,92 @@ def myfunction(*args, **kwargs): ... """ raise AbstractInvokationError('__validate__') + + + +class threaded(DecoratorBase): + """ + Decorate a function to run it in a thread + + Use Case + -------- + Now whenever `myfunction` is invoked, it will be executed in a thread and the `Thread` object will be returned + + >>> @thread.threaded + >>> def myfunction(*args, **kwargs): ... + + >>> myJob = myfunction(1, 2) + >>> type(myjob) + > Thread + + You can also pass keyword arguments to change the thread behaviour, it otherwise follows the defaults of `thread.Thread` + >>> @thread.threaded(daemon = True) + >>> def myotherfunction(): ... + """ + + def __init__( + self, + func: Optional[TargetFunction] = None, + *, + args: Sequence[Data_In] = (), + kwargs: Mapping[str, Data_In] = {}, + ignore_errors: Sequence[type[Exception]] = (), + suppress_errors: bool = False, + **overflow_kwargs: Overflow_In + ) -> None: + """ + Decorate a function to ru nit in a thread + + Parameters + :param func: Automatically passed in + :param args: + """ + + compiled: Mapping[str, Any] = dict( + args = args, + kwargs = kwargs, + ignore_errors = ignore_errors, + suppress_errors = suppress_errors, + **overflow_kwargs + ) + + # Validate arguments + allowed_params = inspect.signature(Thread.__init__).parameters + for key in compiled: + if key not in allowed_params: + raise ArgumentValidationError(f'{key} is not a keyword argument for thread.Thread') + super().__init__(**compiled) if func is None else super().__init__(func, **compiled) + + + def __call__( + self, + func: Optional[TargetFunction] = None, + *args: Any, + **kwargs: Any + ) -> Any: + if func is not None: + args = (func, *args) + return super().__call__(*args, **kwargs) + + + def __validate__(self, *args, **kwargs) -> bool: + return True + + + def __execute__(self, func: TargetFunction, args: Tuple[Any, ...], kwargs: dict[str, Any]) -> Thread: + # Join parsed args and kwargs with decorator defined args and kwargs + parsedKwargs = {} + for i, v in self.kwargs.items(): + if i == 'args': args = ( *v, *args) + elif i == 'kwargs': kwargs[i] = v + else: parsedKwargs[i] = v + + job = Thread( + target = func, + args = args, + kwargs = kwargs, + *self.args, + **parsedKwargs + ) + job.start() + return job From 7d9e39d8946d92396afae9e9c52209b3c520606a Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sat, 16 Dec 2023 00:44:02 +0800 Subject: [PATCH 12/17] Overload decorator implementation --- src/thread/decorators.py | 307 ++++++++++----------------------------- 1 file changed, 79 insertions(+), 228 deletions(-) diff --git a/src/thread/decorators.py b/src/thread/decorators.py index 17c1b3b..fff4de2 100644 --- a/src/thread/decorators.py +++ b/src/thread/decorators.py @@ -5,247 +5,98 @@ """ import inspect -from functools import wraps +from functools import wraps, partial from abc import ABC, abstractmethod -from ._types import TargetFunction, Overflow_In, Data_Out, Data_In -from typing import Any, Callable, Mapping, Tuple, Sequence, Optional, Union +from ._types import Overflow_In, Data_Out, Data_In +from typing import Any, Callable, Mapping, Tuple, Sequence, Optional, Union, Protocol, Iterable, overload +from typing_extensions import ParamSpec, TypeVar, Concatenate from .thread import Thread from .exceptions import AbstractInvokationError, ArgumentValidationError -class DecoratorBase(ABC): +T = TypeVar('T') +P = ParamSpec('P') +TargetFunction = Callable[P, Data_Out] +NoParamReturn = Callable[P, Thread] +WithParamReturn = Callable[[TargetFunction], NoParamReturn] +FullParamReturn = Callable[P, Thread] +WrappedWithParamReturn = Callable[[TargetFunction], WithParamReturn] + + +@overload +def threaded(__function: TargetFunction) -> NoParamReturn: ... + +@overload +def threaded( + *, + args: Sequence[Data_In] = (), + kwargs: Mapping[str, Data_In] = {}, + ignore_errors: Sequence[type[Exception]] = (), + suppress_errors: bool = False, + **overflow_kwargs: Overflow_In +) -> WithParamReturn: ... + +@overload +def threaded( + __function: Callable[P, Data_Out], + *, + args: Sequence[Data_In] = (), + kwargs: Mapping[str, Data_In] = {}, + ignore_errors: Sequence[type[Exception]] = (), + suppress_errors: bool = False, + **overflow_kwargs: Overflow_In +) -> FullParamReturn: ... + + +def threaded( + __function: Optional[TargetFunction] = None, + *, + args: Sequence[Data_In] = (), + kwargs: Mapping[str, Data_In] = {}, + ignore_errors: Sequence[type[Exception]] = (), + suppress_errors: bool = False, + **overflow_kwargs: Overflow_In +) -> Union[NoParamReturn, WithParamReturn, FullParamReturn]: """ - Decorator Base Class - - This should only be used from inheriting classes - - Use Case - -------- - ```py - class MyDecorator(DecoratorBase): - # DocString here for decorator without arguments - - # Arguments to the decorator are here - args: Tuple[Any, ...] - kwargs: Mapping[str, Any] - - def __init__(self, *args, **kwargs): - # DocString here for decorator with arguments - # If you want to have type hinting - # If you want to validate arguments to decorator - super().__init__(*args, **kwargs) - - def __call__(self, *args, **kwargs): - return super().__call__(*args, **kwargs) - - def __validate__(self, args, kwargs): - # If you want to validate arguments to the wrapped function - return bool - - def __execute__(self, func, args, kwargs): - return func(*args, **kwargs) - ``` + test 1 """ - args: Tuple[Any, ...] - kwargs: Mapping[str, Any] - - def __init__(self, *args: Any, **kwargs: Any) -> None: - """ - Decorator Arguments - - There arguments will `NOT` be parsed to the wrapped function - - Parameters - ---------- - :param *: Positional arguments - :param **: Keyword arguments - """ - self.args = args - self.kwargs = kwargs - - def __call__(self, *args: Any, **kwargs: Any) -> Union[Any, Callable[..., Any]]: - """ - The main logic to allow decorators to be invoked: - - ```py - @MyDecorator - def a(): ... - - @MyDecorator() - def b(): ... - - @MyDecorator(1, 2, 3) - def c(): ... - ``` - """ - is_callable = (len(self.args) >= 1) and callable(self.args[0]) - if is_callable and self.__validate__(args, kwargs): - func = self.args[0] - return self.__execute__(func, args, kwargs) - - elif not is_callable: - func = args[0] - - @wraps(self.__execute__) - def wrapper(*args, **kwargs): - if self.__validate__(args, kwargs): - return self.__execute__(func, args, kwargs) - else: - raise ArgumentValidationError() - return wrapper - - else: - raise ArgumentValidationError() - - - @abstractmethod - def __execute__(self, func: Callable[..., Data_Out], args: Tuple[Any, ...], kwargs: Mapping[str, Any]): - """ - The main code returning the result of the wrapped method - - Parameters - ---------- - :param func: The wrapped function - :param args: The tuple of parsed arguments - :param kwargs: The tuple of parsed keyword arguments - - Returns - ------- - :returns Data_Out: The result of the wrapped function - - Raises - ------ - AbstractInvocationsError: When the base class abstract method is invoked - - Use Case - -------- - ```py - class MyDecorator(DecoratorBase): - ... - def __execute__(self, func, args, kwargs): - return func(*args, **kwargs) - ``` - """ - raise AbstractInvokationError('__execute__') - - @abstractmethod - def __validate__(self, args: Tuple[Any, ...], kwargs: Mapping[str, Any]) -> bool: - """ - Validate arguments parsed to the wrapped method - - Parameters - ---------- - :param args: Tuple of parsed positional arguments - :param kwargs: Dictionary of parsed keyword arguments - - Returns - ------- - :returns bool: True if the arguments are valid - - Raises - ------ - AbstractInvokationError: When the base class abstract method is invoked - - Use Case - -------- - ```py - @MyDecorator - def myfunction(*args, **kwargs): ... - - myfunction(1, 2, 3) # Arguments parsed here is validated here - ``` - """ - raise AbstractInvokationError('__validate__') - - - - -class threaded(DecoratorBase): - """ - Decorate a function to run it in a thread - - Use Case - -------- - Now whenever `myfunction` is invoked, it will be executed in a thread and the `Thread` object will be returned - - >>> @thread.threaded - >>> def myfunction(*args, **kwargs): ... - - >>> myJob = myfunction(1, 2) - >>> type(myjob) - > Thread - - You can also pass keyword arguments to change the thread behaviour, it otherwise follows the defaults of `thread.Thread` - >>> @thread.threaded(daemon = True) - >>> def myotherfunction(): ... - """ - - def __init__( - self, - func: Optional[TargetFunction] = None, - *, - args: Sequence[Data_In] = (), - kwargs: Mapping[str, Data_In] = {}, - ignore_errors: Sequence[type[Exception]] = (), - suppress_errors: bool = False, - **overflow_kwargs: Overflow_In - ) -> None: - """ - Decorate a function to ru nit in a thread - - Parameters - :param func: Automatically passed in - :param args: - """ - - compiled: Mapping[str, Any] = dict( - args = args, - kwargs = kwargs, - ignore_errors = ignore_errors, - suppress_errors = suppress_errors, - **overflow_kwargs - ) - - # Validate arguments - allowed_params = inspect.signature(Thread.__init__).parameters - for key in compiled: - if key not in allowed_params: - raise ArgumentValidationError(f'{key} is not a keyword argument for thread.Thread') - super().__init__(**compiled) if func is None else super().__init__(func, **compiled) - - - def __call__( - self, - func: Optional[TargetFunction] = None, - *args: Any, - **kwargs: Any - ) -> Any: - if func is not None: - args = (func, *args) - return super().__call__(*args, **kwargs) - - - def __validate__(self, *args, **kwargs) -> bool: - return True + if not callable(__function): + def wrapper(func: TargetFunction) -> FullParamReturn: + return threaded( + func, + args = args, + kwargs = kwargs, + ignore_errors = ignore_errors, + suppress_errors = suppress_errors, + **overflow_kwargs + ) + return wrapper + + overflow_kwargs.update({ + 'ignore_errors': ignore_errors, + 'suppress_errors': suppress_errors + }) + + kwargs = dict(kwargs) - - def __execute__(self, func: TargetFunction, args: Tuple[Any, ...], kwargs: dict[str, Any]) -> Thread: - # Join parsed args and kwargs with decorator defined args and kwargs - parsedKwargs = {} - for i, v in self.kwargs.items(): - if i == 'args': args = ( *v, *args) - elif i == 'kwargs': kwargs[i] = v - else: parsedKwargs[i] = v + @wraps(__function) + def wrapped(*parsed_args: P.args, **parsed_kwargs: P.kwargs) -> Thread: + """test 2""" + kwargs.update(parsed_kwargs) + + processed_args = ( *args, *parsed_args ) + processed_kwargs = { i:v for i,v in kwargs.items() if i not in ['args', 'kwargs'] } job = Thread( - target = func, - args = args, - kwargs = kwargs, - *self.args, - **parsedKwargs + target = __function, + args = processed_args, + kwargs = processed_kwargs, + **overflow_kwargs ) job.start() return job + + return wrapped From 63f4d5d7481b86b8c7da4915f1b80a38f9c5f602 Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sat, 16 Dec 2023 01:25:57 +0800 Subject: [PATCH 13/17] Updated thread.threaded implementation --- src/thread/decorators.py | 55 ++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/thread/decorators.py b/src/thread/decorators.py index fff4de2..0d12d50 100644 --- a/src/thread/decorators.py +++ b/src/thread/decorators.py @@ -4,24 +4,20 @@ Documentation: https://thread.ngjx.org """ -import inspect -from functools import wraps, partial -from abc import ABC, abstractmethod +from functools import wraps +from .thread import Thread from ._types import Overflow_In, Data_Out, Data_In -from typing import Any, Callable, Mapping, Tuple, Sequence, Optional, Union, Protocol, Iterable, overload +from typing import Any, Callable, Mapping, Sequence, Optional, Union, overload from typing_extensions import ParamSpec, TypeVar, Concatenate -from .thread import Thread -from .exceptions import AbstractInvokationError, ArgumentValidationError - T = TypeVar('T') P = ParamSpec('P') TargetFunction = Callable[P, Data_Out] NoParamReturn = Callable[P, Thread] WithParamReturn = Callable[[TargetFunction], NoParamReturn] -FullParamReturn = Callable[P, Thread] +FullParamReturn = Callable[Concatenate[TargetFunction, ...], Thread] WrappedWithParamReturn = Callable[[TargetFunction], WithParamReturn] @@ -60,7 +56,43 @@ def threaded( **overflow_kwargs: Overflow_In ) -> Union[NoParamReturn, WithParamReturn, FullParamReturn]: """ - test 1 + Decorate a function to run it in a thread + + Parameters + ---------- + :param __function: The function to run in a thread + :param args: Keyword-Only arguments to pass into `thread.Thread` + :param kwargs: Keyword-Only keyword arguments to pass into `thread.Thread` + :param ignore_errors: Keyword-Only arguments to pass into `thread.Thread` + :param suppress_errors: Keyword-Only arguments to pass into `thread.Thread` + :param **: Keyword-Only arguments to pass into `thread.Thread` + + Returns + ------- + :return decorator: + + Use Case + -------- + Now whenever `myfunction` is invoked, it will be executed in a thread and the `Thread` object will be returned + + >>> @thread.threaded + >>> def myfunction(*args, **kwargs): ... + + >>> myJob = myfunction(1, 2) + >>> type(myjob) + > Thread + + You can also pass keyword arguments to change the thread behaviour, it otherwise follows the defaults of `thread.Thread` + >>> @thread.threaded(daemon = True) + >>> def myfunction(): ... + + Args will be ordered infront of function-parsed args parsed into `thread.Thread.args` + >>> @thread.threaded(args = (1)) + >>> def myfunction(*args): + >>> print(args) + >>> + >>> myfunction(4, 6).get_return_value() + 1, 4, 6 """ if not callable(__function): @@ -84,11 +116,10 @@ def wrapper(func: TargetFunction) -> FullParamReturn: @wraps(__function) def wrapped(*parsed_args: P.args, **parsed_kwargs: P.kwargs) -> Thread: - """test 2""" kwargs.update(parsed_kwargs) processed_args = ( *args, *parsed_args ) - processed_kwargs = { i:v for i,v in kwargs.items() if i not in ['args', 'kwargs'] } + processed_kwargs = { i:v for i,v in parsed_kwargs.items() if i not in ['args', 'kwargs'] } job = Thread( target = __function, @@ -100,3 +131,5 @@ def wrapped(*parsed_args: P.args, **parsed_kwargs: P.kwargs) -> Thread: return job return wrapped + + From 532d8435364dd205e6836138e4cd7be57ae7bd2f Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sat, 16 Dec 2023 01:26:56 +0800 Subject: [PATCH 14/17] + Added typing_extensions package --- poetry.lock | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 68df6df..044e16a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -316,4 +316,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "43238a4a9b3c3d7aa608a6cbd6a95dd47889b7edfeb392db46529c2389c9c120" +content-hash = "6a92a75c564698cb87dc8e31a9df0b685a262c532cc0637fd1f90a91a0106cfe" diff --git a/pyproject.toml b/pyproject.toml index 9dbec22..c771efd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ thread = "thread.__main__:app" [tool.poetry.dependencies] python = "^3.9" typer = {extras = ["all"], version = "^0.9.0"} +typing-extensions = "^4.9.0" [tool.poetry.group.dev.dependencies] From 3ba364909b5ebc12105fc55de21872bba2417bdc Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sat, 16 Dec 2023 01:38:02 +0800 Subject: [PATCH 15/17] + Added test cases for decorators --- tests/test_decorator.py | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/test_decorator.py diff --git a/tests/test_decorator.py b/tests/test_decorator.py new file mode 100644 index 0000000..53117d2 --- /dev/null +++ b/tests/test_decorator.py @@ -0,0 +1,55 @@ +import time +import pytest +from src.thread import threaded, exceptions + + +# >>>>>>>>>> Dummy Functions <<<<<<<<<< # +def _dummy_target_raiseToPower(x: float, power: float, delay: float = 0): + time.sleep(delay) + return x**power + +def _dummy_raiseException(x: Exception, delay: float = 0): + time.sleep(delay) + raise x + +def _dummy_iterative(itemCount: int, pTime: float = 0.1, delay: float = 0): + time.sleep(delay) + for i in range(itemCount): + time.sleep(pTime) + + + + +# >>>>>>>>>> General Use <<<<<<<<<< # +def test_creationNoParam(): + @threaded + def _run(*args): + return _dummy_target_raiseToPower(*args) + + x = _run(2, 2) + assert x.get_return_value() == 4 + +def test_creationEmptyParam(): + @threaded() + def _run(*args): + return _dummy_target_raiseToPower(*args) + + x = _run(2, 2) + assert x.get_return_value() == 4 + +def test_creationWithParam(): + @threaded(daemon = True) + def _run(*args): + return _dummy_target_raiseToPower(*args) + + x = _run(2, 2) + assert x.daemon + assert x.get_return_value() == 4 + +def test_argJoin(): + @threaded(daemon = True, args = (1, 2, 3)) + def _run(*args): + return args + + x = _run(8, 9) + assert x.get_return_value() == (1, 2, 3, 8, 9) From 3415715d9fcc03498a865d77afb917afd396166f Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sat, 16 Dec 2023 01:38:42 +0800 Subject: [PATCH 16/17] - Removed unused exception --- src/thread/exceptions.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index b807769..881dee3 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -52,20 +52,3 @@ def __init__(self, message: Optional[str] = '', extra: Sequence[Tuple[Exception, new_message += f'{trace}\n{v[0]}' new_message += '<<<<<<<<<<' super().__init__(new_message) - - - -# DECORATOR ERRORS # -class AbstractInvokationError(ErrorBase): - """Exception class for attempting to invoke an abstract method which is only accessible from inheriting classes""" - message: str = 'Attempt to invoke abstract method [{method_name}]!' - - def __init__(self, method_name: str) -> None: - super().__init__(self.message.format(method_name = method_name)) - -class ArgumentValidationError(ErrorBase): - """Exception class for when validating arguments passed to the wrapped method fails""" - message: str = 'Validation for arguments passed to the wrapped method failed' - - def __init__(self, additional: Optional[str] = '') -> None: - super().__init__(self.message + f'\n{additional}') From 703a82b3081c7808dee746988500bd6d61dd0d0b Mon Sep 17 00:00:00 2001 From: AlexNg Date: Sat, 16 Dec 2023 01:44:45 +0800 Subject: [PATCH 17/17] - Removed Concatenate typing --- src/thread/decorators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/thread/decorators.py b/src/thread/decorators.py index 0d12d50..4755d3c 100644 --- a/src/thread/decorators.py +++ b/src/thread/decorators.py @@ -8,8 +8,8 @@ from .thread import Thread from ._types import Overflow_In, Data_Out, Data_In -from typing import Any, Callable, Mapping, Sequence, Optional, Union, overload -from typing_extensions import ParamSpec, TypeVar, Concatenate +from typing import Callable, Mapping, Sequence, Optional, Union, overload +from typing_extensions import ParamSpec, TypeVar T = TypeVar('T') @@ -17,7 +17,7 @@ TargetFunction = Callable[P, Data_Out] NoParamReturn = Callable[P, Thread] WithParamReturn = Callable[[TargetFunction], NoParamReturn] -FullParamReturn = Callable[Concatenate[TargetFunction, ...], Thread] +FullParamReturn = Callable[P, Thread] WrappedWithParamReturn = Callable[[TargetFunction], WithParamReturn]