diff --git a/.github/CODESTYLE.md b/.github/CODESTYLE.md index dfeb7a4..9499611 100644 --- a/.github/CODESTYLE.md +++ b/.github/CODESTYLE.md @@ -57,21 +57,27 @@ class ExampleClass: ## Commit Message Guidelines -When committing, commit messages are prefixed with a `+` or `-`. Depending on the type of change made -influences which prefix is used. - - - `+` when something is added. - - `-` when something is removed. - - none: when neither is applicable, like merge commits. +When committing, commit messages are prefixed with one of the following depending on the type of change made. + + - `feat:` when a new feature is introduced with the changes. + - `fix:` when a bug fix has occurred. + - `chore:` for changes that do not relate to a fix or feature and do not modify *source* or *tests*. (like updating dependencies) + - `refactor:` for refactoring code that neither fixes a bug nor adds a feature. + - `docs:` when changes are made to documentation. + - `style:` when changes that do not affect the code, but modify formatting. + - `test:` when changes to tests are made. + - `perf:` for changes that improve performance. + - `ci:` for changes that affect CI. + - `build:` for changes that affect the build system or external dependencies. + - `revert:` when reverting changes. Commit messages are also to begin with an uppercase character. Below list some example commit messages. +```sh +git commit -m "docs: Added README.md" +git commit -m "revert: Removed README.md" +git commit -m "docs: Moved README.md" ``` -git commit -m "+ Added README.md" -git commit -m "- Removed README.md" -git commit -m "Moved README.md" -``` - ## Markdown Guidelines diff --git a/.github/ISSUE_TEMPLATE/8-version-checklist.md b/.github/ISSUE_TEMPLATE/8-version-checklist.md new file mode 100644 index 0000000..f973413 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/8-version-checklist.md @@ -0,0 +1,14 @@ +--- +name: "vX.X.X: Version Release Checklist" +about: "Checklist for version release" +labels: "Documentation" +assignees: caffeine-addictt + +--- + +# Version Release Checklist + +- [ ] I have updated the README.md file +- [ ] I have ensured that all tests pass +- [ ] I have incremented the version number in `__init__.py` +- [ ] I have incremented the version number in `pyproject.toml` diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..03f3266 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,40 @@ +name-template: "v$RESOLVED_VERSION" +tag-template: "v$RESOLVED_VERSION" +template: | + # What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION + +categories: + - title: "New" + label: "Type: Feature" + + - title: "Bug Fixes" + label: "Type: Bug" + + - title: "Improvements" + label: "Type: Enhancement" + + - title: "Other changes" + + - title: "Documentation" + label: "Documentation" + collapse-after: 5 + +version-resolver: + major: + labels: + - "Type: Breaking" + minor: + labels: + - "Type: Feature" + patch: + labels: + - "Type: Bug" + - "Documentation" + - "Type: Security" + +exclude-labels: + - "Skip-Changelog" diff --git a/.github/settings.yml b/.github/settings.yml index 40c606e..033ab60 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -1,61 +1,65 @@ labels: - - name: 'Type: Bug' + - name: "Type: Breaking" + color: a90000 + description: A problem or enhancement related to a breaking change. + + - name: "Type: Bug" color: e80c0c description: Something isn't working as expected. - - name: 'Type: Enhancement' + - name: "Type: Enhancement" color: 54b2ff description: Suggest an improvement for an existing feature. - - name: 'Type: Feature' + - name: "Type: Feature" color: 54b2ff description: Suggest a new feature. - - name: 'Type: Security' + - name: "Type: Security" color: fbff00 description: A problem or enhancement related to a security issue. - - name: 'Type: Question' + - name: "Type: Question" color: 9309ab description: Request for information. - - name: 'Type: Test' + - name: "Type: Test" color: ce54e3 description: A problem or enhancement related to a test. - - name: 'Status: Awaiting Review' + - name: "Status: Awaiting Review" color: 24d15d description: Ready for review. - - name: 'Status: WIP' + - name: "Status: WIP" color: 07b340 description: Currently being worked on. - - name: 'Status: Waiting' + - name: "Status: Waiting" color: 38C968 description: Waiting on something else to be ready. - - name: 'Status: Stale' + - name: "Status: Stale" color: 66b38a description: Has had no activity for some time. - - name: 'Duplicate' + - name: "Duplicate" color: EB862D description: Duplicate of another issue. - - name: 'Invalid' + - name: "Invalid" color: faef50 description: This issue doesn't seem right. - - name: 'Priority: High +' + - name: "Priority: High +" color: ff008c description: Task is considered higher-priority. - - name: 'Priority: Low -' + - name: "Priority: Low -" color: 690a34 description: Task is considered lower-priority. - - name: 'Documentation' + - name: "Documentation" color: 2fbceb description: An issue/change with the documentation. @@ -63,18 +67,22 @@ labels: color: C8D9E6 description: Reported issue is working as intended. - - name: '3rd party issue' + - name: "3rd party issue" color: e88707 description: This issue might be caused by a 3rd party script/package/other reasons - - name: 'Os: Windows' + - name: "Os: Windows" color: AEB1C2 description: Is Windows-specific - - name: 'Os: Mac' + - name: "Os: Mac" color: AEB1C2 description: Is Mac-specific - - name: 'Os: Linux' + - name: "Os: Linux" color: AEB1C2 description: Is Linux-specific + + - name: "Skip-Changelog" + color: AEB1C2 + description: Skip changelog in release tag diff --git a/.github/workflows/draft-release-tag.yml b/.github/workflows/draft-release-tag.yml new file mode 100644 index 0000000..ecf945f --- /dev/null +++ b/.github/workflows/draft-release-tag.yml @@ -0,0 +1,24 @@ +name: Release Drafter + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + update_release_draft: + permissions: + contents: write + pull-requests: read + + runs-on: ubuntu-latest + steps: + # Drafts your next Release notes as Pull Requests are merged into "master" + - uses: release-drafter/release-drafter@v5 + with: + disable-autolabeler: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/poetry.lock b/poetry.lock index 044e16a..71c0215 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,19 +1,5 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. -[[package]] -name = "click" -version = "8.1.7" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - [[package]] name = "colorama" version = "0.4.6" @@ -114,41 +100,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "markdown-it-py" -version = "3.0.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.8" -files = [ - {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, - {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - [[package]] name = "packaging" version = "23.2" @@ -175,21 +126,6 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "pygments" -version = "2.17.2" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, - {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, -] - -[package.extras] -plugins = ["importlib-metadata"] -windows-terminal = ["colorama (>=0.4.6)"] - [[package]] name = "pytest" version = "7.4.3" @@ -212,24 +148,6 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] -[[package]] -name = "rich" -version = "13.7.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, - {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - [[package]] name = "ruff" version = "0.1.7" @@ -256,17 +174,6 @@ files = [ {file = "ruff-0.1.7.tar.gz", hash = "sha256:dffd699d07abf54833e5f6cc50b85a6ff043715da8788c4a79bcd4ab4734d306"}, ] -[[package]] -name = "shellingham" -version = "1.5.4" -description = "Tool to Detect Surrounding Shell" -optional = false -python-versions = ">=3.7" -files = [ - {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, - {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -278,30 +185,6 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -[[package]] -name = "typer" -version = "0.9.0" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.6" -files = [ - {file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee"}, - {file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2"}, -] - -[package.dependencies] -click = ">=7.1.1,<9.0.0" -colorama = {version = ">=0.4.3,<0.5.0", optional = true, markers = "extra == \"all\""} -rich = {version = ">=10.11.0,<14.0.0", optional = true, markers = "extra == \"all\""} -shellingham = {version = ">=1.3.0,<2.0.0", optional = true, markers = "extra == \"all\""} -typing-extensions = ">=3.7.4.3" - -[package.extras] -all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] -doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] -test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] - [[package]] name = "typing-extensions" version = "4.9.0" @@ -316,4 +199,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "6a92a75c564698cb87dc8e31a9df0b685a262c532cc0637fd1f90a91a0106cfe" +content-hash = "c2e130ba9f37143508e38333da3c7d3a69c2429a321398834d5f6ab6ef3e5959" diff --git a/pyproject.toml b/pyproject.toml index c771efd..0adf041 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,19 +5,26 @@ description = "Threading module extension" authors = ["Alex "] license = "BSD-3-Clause" readme = "README.md" -packages = [{include = "thread", from = "src"}] +packages = [ + { include = "thread", from = "src" }, + { include = "thread/py.typed", from = "src" }, +] include = [{ path = "tests", format = "sdist" }] homepage = "https://github.com/python-thread/thread" repository = "https://github.com/python-thread/thread" documentation = "https://github.com/python-thread/thread/blob/main/docs/getting-started.md" -keywords = ["threading", "extension", "multiprocessing"] +keywords = ["thread", "threading", "extension", "multiprocessing"] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License" + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", ] [tool.poetry.urls] +Changelog = "https://github.com/python-thread/thread/releases" "Bug Tracker" = "https://github.com/python-thread/thread/issues" [tool.poetry.scripts] @@ -25,7 +32,6 @@ thread = "thread.__main__:app" [tool.poetry.dependencies] python = "^3.9" -typer = {extras = ["all"], version = "^0.9.0"} typing-extensions = "^4.9.0" diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..12673e1 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,52 @@ +indent-width = 2 + +[format] +# Exclude commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +indent-style = "space" +line-ending = "lf" +quote-style = "single" +docstring-code-format = true + + +[lint] +# Avoid enforcing line-length violations (`E501`) +ignore = ["E501"] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + + +# Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. +[lint.per-file-ignores] +"__init__.py" = ["E402", "F401"] +"**/__init__.py" = ["E402", "F401"] +"**/{tests,docs,tools}/*" = ["E402"] diff --git a/src/thread/__init__.py b/src/thread/__init__.py index 3a77a8c..1bb4a11 100644 --- a/src/thread/__init__.py +++ b/src/thread/__init__.py @@ -19,24 +19,27 @@ # Export Core -from .thread import ( - Thread, - ParallelProcessing -) +from .thread import Thread, ParallelProcessing -from . import ( - _types as types, - exceptions -) +from . import _types as types, exceptions # Export decorators -from .decorators import ( - threaded, - processor -) +from .decorators import threaded, processor # Configuration from .utils import Settings + + +# Wildcard Export +__all__ = [ + 'Thread', + 'ParallelProcessing', + 'threaded', + 'processor', + 'types', + 'exceptions', + 'Settings', +] diff --git a/src/thread/__main__.py b/src/thread/__main__.py index 5a54980..4dd94a0 100644 --- a/src/thread/__main__.py +++ b/src/thread/__main__.py @@ -1,6 +1,5 @@ - # To make CLI accessible with py/python/python3 -m thread ... from .cli import app if __name__ == '__main__': - app(prog_name = 'thread') + app(prog_name='thread') diff --git a/src/thread/_types.py b/src/thread/_types.py index 2937ef1..8040d78 100644 --- a/src/thread/_types.py +++ b/src/thread/_types.py @@ -16,14 +16,13 @@ # Variable Types ThreadStatus = Literal[ - 'Idle', - 'Running', - 'Invoking hooks', - 'Completed', - - 'Errored', - 'Kill Scheduled', - 'Killed' +'Idle', +'Running', +'Invoking hooks', +'Completed', +'Errored', +'Kill Scheduled', +'Killed', ] diff --git a/src/thread/cli.py b/src/thread/cli.py index 63847a5..d8c5c26 100644 --- a/src/thread/cli.py +++ b/src/thread/cli.py @@ -1,9 +1,10 @@ - try: import importlib + thread_cli = importlib.import_module('thread-cli') app = thread_cli.app except ModuleNotFoundError: - def app(prog_name = 'thread'): + + def app(prog_name='thread'): print('thread-cli not found, please install it with `pip install thread-cli`') - exit(1) \ No newline at end of file + exit(1) diff --git a/src/thread/decorators/_processor.py b/src/thread/decorators/_processor.py index b7024e5..3511575 100644 --- a/src/thread/decorators/_processor.py +++ b/src/thread/decorators/_processor.py @@ -18,15 +18,26 @@ TargetFunction = Callable[Concatenate[_DataT, _TargetP], _TargetT] -NoParamReturn = Callable[Concatenate[Sequence[_DataT], _TargetP], ParallelProcessing[_TargetP, _TargetT, _DataT]] -WithParamReturn = Callable[[TargetFunction[_DataT, _TargetP, _TargetT]], NoParamReturn[_DataT, _TargetP, _TargetT]] -FullParamReturn = Callable[Concatenate[Sequence[_DataT], _TargetP], ParallelProcessing[_TargetP, _TargetT, _DataT]] - - +NoParamReturn = Callable[ +Concatenate[Sequence[_DataT], _TargetP], +ParallelProcessing[_TargetP, _TargetT, _DataT], +] +WithParamReturn = Callable[ +[TargetFunction[_DataT, _TargetP, _TargetT]], +NoParamReturn[_DataT, _TargetP, _TargetT], +] +FullParamReturn = Callable[ +Concatenate[Sequence[_DataT], _TargetP], +ParallelProcessing[_TargetP, _TargetT, _DataT], +] @overload -def processor(__function: TargetFunction[_DataT, _TargetP, _TargetT]) -> NoParamReturn[_DataT, _TargetP, _TargetT]: ... +def processor( + __function: TargetFunction[_DataT, _TargetP, _TargetT], + ) -> NoParamReturn[_DataT, _TargetP, _TargetT]: + ... + @overload def processor( @@ -35,8 +46,10 @@ def processor( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - **overflow_kwargs: Overflow_In -) -> WithParamReturn[_DataT, _TargetP, _TargetT]: ... + **overflow_kwargs: Overflow_In, + ) -> WithParamReturn[_DataT, _TargetP, _TargetT]: + ... + @overload def processor( @@ -46,8 +59,9 @@ def processor( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - **overflow_kwargs: Overflow_In -) -> FullParamReturn[_DataT, _TargetP, _TargetT]: ... + **overflow_kwargs: Overflow_In, + ) -> FullParamReturn[_DataT, _TargetP, _TargetT]: + ... def processor( @@ -57,8 +71,12 @@ def processor( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - **overflow_kwargs: Overflow_In -) -> Union[NoParamReturn[_DataT, _TargetP, _TargetT], WithParamReturn[_DataT, _TargetP, _TargetT], FullParamReturn[_DataT, _TargetP, _TargetT]]: + **overflow_kwargs: Overflow_In, + ) -> Union[ + NoParamReturn[_DataT, _TargetP, _TargetT], + WithParamReturn[_DataT, _TargetP, _TargetT], + FullParamReturn[_DataT, _TargetP, _TargetT], +]: """ Decorate a function to run it in a thread @@ -88,7 +106,8 @@ def processor( You can also pass keyword arguments to change the thread behaviour, it otherwise follows the defaults of `thread.Thread` >>> @thread.threaded(daemon = True) - >>> def myfunction(): ... + >>> def myfunction(): + ... ... Args will be ordered infront of function-parsed args parsed into `thread.Thread.args` >>> @thread.threaded(args = (1)) @@ -100,39 +119,46 @@ def processor( """ if not callable(__function): - def wrapper(func: TargetFunction[_DataT, _TargetP, _TargetT]) -> FullParamReturn[_DataT, _TargetP, _TargetT]: + + def wrapper( + func: TargetFunction[_DataT, _TargetP, _TargetT], + ) -> FullParamReturn[_DataT, _TargetP, _TargetT]: return processor( func, - args = args, - kwargs = kwargs, - ignore_errors = ignore_errors, - suppress_errors = suppress_errors, - **overflow_kwargs + args=args, + kwargs=kwargs, + ignore_errors=ignore_errors, + suppress_errors=suppress_errors, + **overflow_kwargs, ) + return wrapper - overflow_kwargs.update({ - 'ignore_errors': ignore_errors, - 'suppress_errors': suppress_errors - }) + overflow_kwargs.update( + {'ignore_errors': ignore_errors, 'suppress_errors': suppress_errors} + ) kwargs = dict(kwargs) - + @wraps(__function) - def wrapped(data: Sequence[_DataT], *parsed_args: _TargetP.args, **parsed_kwargs: _TargetP.kwargs) -> ParallelProcessing[_TargetP, _TargetT, _DataT]: + def wrapped( + data: Sequence[_DataT], + *parsed_args: _TargetP.args, + **parsed_kwargs: _TargetP.kwargs, + ) -> ParallelProcessing[_TargetP, _TargetT, _DataT]: kwargs.update(parsed_kwargs) - processed_args = ( *args, *parsed_args ) - processed_kwargs = { i:v for i,v in kwargs.items() if i not in ['args', 'kwargs'] } + processed_args = (*args, *parsed_args) + processed_kwargs = {i: v for i, v in kwargs.items() if i not in ['args', 'kwargs']} job = ParallelProcessing( - function = __function, - dataset = data, - args = processed_args, - kwargs = processed_kwargs, - **overflow_kwargs + function=__function, + dataset=data, + args=processed_args, + kwargs=processed_kwargs, + **overflow_kwargs, ) job.start() return job - + return wrapped diff --git a/src/thread/decorators/_threaded.py b/src/thread/decorators/_threaded.py index 911b196..4dd49b2 100644 --- a/src/thread/decorators/_threaded.py +++ b/src/thread/decorators/_threaded.py @@ -22,10 +22,10 @@ FullParamReturn = Callable[P, Thread[P, T]] - - @overload -def threaded(__function: TargetFunction[P, T]) -> NoParamReturn[P, T]: ... +def threaded(__function: TargetFunction[P, T]) -> NoParamReturn[P, T]: + ... + @overload def threaded( @@ -34,8 +34,10 @@ def threaded( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - **overflow_kwargs: Overflow_In -) -> WithParamReturn[P, T]: ... + **overflow_kwargs: Overflow_In, + ) -> WithParamReturn[P, T]: + ... + @overload def threaded( @@ -45,8 +47,9 @@ def threaded( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - **overflow_kwargs: Overflow_In -) -> FullParamReturn[P, T]: ... + **overflow_kwargs: Overflow_In, + ) -> FullParamReturn[P, T]: + ... def threaded( @@ -56,8 +59,8 @@ def threaded( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - **overflow_kwargs: Overflow_In -) -> Union[NoParamReturn[P, T], WithParamReturn[P, T], FullParamReturn[P, T]]: + **overflow_kwargs: Overflow_In, + ) -> Union[NoParamReturn[P, T], WithParamReturn[P, T], FullParamReturn[P, T]]: """ Decorate a function to run it in a thread @@ -87,7 +90,8 @@ def threaded( You can also pass keyword arguments to change the thread behaviour, it otherwise follows the defaults of `thread.Thread` >>> @thread.threaded(daemon = True) - >>> def myfunction(): ... + >>> def myfunction(): + ... ... Args will be ordered infront of function-parsed args parsed into `thread.Thread.args` >>> @thread.threaded(args = (1)) @@ -99,38 +103,38 @@ def threaded( """ if not callable(__function): + def wrapper(func: TargetFunction[P, T]) -> FullParamReturn[P, T]: return threaded( func, - args = args, - kwargs = kwargs, - ignore_errors = ignore_errors, - suppress_errors = suppress_errors, - **overflow_kwargs + args=args, + kwargs=kwargs, + ignore_errors=ignore_errors, + suppress_errors=suppress_errors, + **overflow_kwargs, ) + return wrapper - overflow_kwargs.update({ - 'ignore_errors': ignore_errors, - 'suppress_errors': suppress_errors - }) + overflow_kwargs.update( + {'ignore_errors': ignore_errors, 'suppress_errors': suppress_errors} + ) kwargs = dict(kwargs) - + @wraps(__function) def wrapped(*parsed_args: P.args, **parsed_kwargs: P.kwargs) -> Thread[P, T]: kwargs.update(parsed_kwargs) - processed_args = ( *args, *parsed_args ) - processed_kwargs = { i:v for i,v in parsed_kwargs.items() if i not in ['args', 'kwargs'] } + processed_args = (*args, *parsed_args) + processed_kwargs = { + i: v for i, v in parsed_kwargs.items() if i not in ['args', 'kwargs'] + } job = Thread( - target = __function, - args = processed_args, - kwargs = processed_kwargs, - **overflow_kwargs + target=__function, args=processed_args, kwargs=processed_kwargs, **overflow_kwargs ) job.start() return job - + return wrapped diff --git a/src/thread/exceptions.py b/src/thread/exceptions.py index 881dee3..631ee0f 100644 --- a/src/thread/exceptions.py +++ b/src/thread/exceptions.py @@ -10,35 +10,47 @@ class ErrorBase(Exception): """Base exception class for all errors within this library""" + message: str = 'Something went wrong!' + def __init__(self, message: Optional[str] = None, *args: Any, **kwargs: Any) -> None: message = message or self.message super().__init__(message, *args, **kwargs) - # THREAD ERRORS # class ThreadStillRunningError(ErrorBase): - """Exception class for attempting to invoke a method which requries the thread not be running, but isn't""" + """Exception class for attempting to invoke a method which requires the thread not be running, but isn't""" + message: str = 'Thread is still running, unable to invoke method. You can wait for the thread to terminate with `Thread.join()` or check with `Thread.is_alive()`' + class ThreadNotRunningError(ErrorBase): """Exception class for attempting to invoke a method which requires the thread to be running, but isn't""" - message: str = 'Thread is not running, unable to invoke method. Have you ran `Thread.start()`?' + + message: str = ( + 'Thread is not running, unable to invoke method. Have you ran `Thread.start()`?' + ) + class ThreadNotInitializedError(ErrorBase): """Exception class for attempting to invoke a method which requires the thread to be initialized, but isn't""" + message: str = 'Thread is not initialized, unable to invoke method.' + class HookRuntimeError(ErrorBase): """Exception class for hook runtime errors""" + message: str = 'Encountered runtime errors in hooks' count: int = 0 - def __init__(self, message: Optional[str] = '', extra: Sequence[Tuple[Exception, str]] = []) -> None: + def __init__( + self, message: Optional[str] = '', extra: Sequence[Tuple[Exception, str]] = [] + ) -> None: """ Extra for parsing all hooks that errored - + Parameters ---------- :param message: The message to be parsed, can be left blank diff --git a/src/thread/py.typed b/src/thread/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/thread/thread.py b/src/thread/thread.py index 861e958..a0017a6 100644 --- a/src/thread/thread.py +++ b/src/thread/thread.py @@ -20,20 +20,24 @@ class ParallelProcessing: ... from .utils.algorithm import chunk_split from ._types import ( - ThreadStatus, Data_In, Data_Out, Overflow_In, - TargetFunction, _Target_P, _Target_T, - DatasetFunction, _Dataset_T, - HookFunction + ThreadStatus, + Data_In, + Data_Out, + Overflow_In, + TargetFunction, + _Target_P, + _Target_T, + DatasetFunction, + _Dataset_T, + HookFunction, ) from typing_extensions import Generic, ParamSpec -from typing import ( - List, - Callable, Optional, Union, - Mapping, Sequence, Tuple -) +from typing import List, Callable, Optional, Union, Mapping, Sequence, Tuple Threads: set['Thread'] = set() + + class Thread(threading.Thread, Generic[_Target_P, _Target_T]): """ Wraps python's `threading.Thread` class @@ -42,18 +46,17 @@ class Thread(threading.Thread, Generic[_Target_P, _Target_T]): Type-Safe and provides more functionality on top """ - status : ThreadStatus - hooks : List[HookFunction] + status: ThreadStatus + hooks: List[HookFunction] _returned_value: Data_Out - errors : List[Exception] - ignore_errors : Sequence[type[Exception]] + errors: List[Exception] + ignore_errors: Sequence[type[Exception]] suppress_errors: bool # threading.Thread stuff - _initialized : bool - _run : Callable - + _initialized: bool + _run: Callable def __init__( self, @@ -62,13 +65,12 @@ def __init__( kwargs: Mapping[str, Data_In] = {}, ignore_errors: Sequence[type[Exception]] = (), suppress_errors: bool = False, - name: Optional[str] = None, daemon: bool = False, - group = None, + group=None, *overflow_args: Overflow_In, - **overflow_kwargs: Overflow_In - ) -> None: + **overflow_kwargs: Overflow_In, + ) -> None: """ Initializes a thread @@ -76,7 +78,7 @@ def __init__( ---------- :param target: This should be a function that takes in anything and returns anything :param args: This should be an interable sequence of arguments parsed to the `target` function (e.g. tuple('foo', 'bar')) - :param kwargs: This should be the kwargs pased to the `target` function (e.g. dict(foo = 'bar')) + :param kwargs: This should be the kwargs parsed to the `target` function (e.g. dict(foo = 'bar')) :param ignore_errors: This should be an interable sequence of all exceptions to ignore. To ignore all exceptions, parse tuple(Exception) :param suppress_errors: This should be a boolean indicating whether exceptions will be raised, else will only write to internal `errors` property :param name: This is an argument parsed to `threading.Thread` @@ -95,21 +97,25 @@ def __init__( self.suppress_errors = suppress_errors super().__init__( - target = _target, - args = args, - kwargs = kwargs, - name = name, - daemon = daemon, - group = group, + target=_target, + args=args, + kwargs=kwargs, + name=name, + daemon=daemon, + group=group, *overflow_args, - **overflow_kwargs + **overflow_kwargs, ) - - def _wrap_target(self, target: TargetFunction[_Target_P, _Target_T]) -> TargetFunction[_Target_P, Union[_Target_T, None]]: + def _wrap_target( + self, target: TargetFunction[_Target_P, _Target_T] + ) -> TargetFunction[_Target_P, Union[_Target_T, None]]: """Wraps the target function""" + @wraps(target) - def wrapper(*args: _Target_P.args, **kwargs: _Target_P.kwargs) -> Union[_Target_T, None]: + def wrapper( + *args: _Target_P.args, **kwargs: _Target_P.kwargs + ) -> Union[_Target_T, None]: self.status = 'Running' global Threads @@ -122,13 +128,13 @@ def wrapper(*args: _Target_P.args, **kwargs: _Target_P.kwargs) -> Union[_Target_ self.status = 'Errored' self.errors.append(e) return - + self.status = 'Invoking hooks' self._invoke_hooks() Threads.remove(self) self.status = 'Completed' + return wrapper - def _invoke_hooks(self) -> None: """Invokes hooks in the thread""" @@ -138,54 +144,48 @@ def _invoke_hooks(self) -> None: hook(self._returned_value) except Exception as e: if not any(isinstance(e, ignore) for ignore in self.ignore_errors): - errors.append(( - e, - hook.__name__ - )) + errors.append((e, hook.__name__)) if len(errors) > 0: - self.errors.append(exceptions.HookRuntimeError( - None, errors - )) - + self.errors.append(exceptions.HookRuntimeError(None, errors)) def _handle_exceptions(self) -> None: """Raises exceptions if not suppressed in the main thread""" if self.suppress_errors: return - + for e in self.errors: raise e - def global_trace(self, frame, event: str, arg) -> Optional[Callable]: if event == 'call': return self.local_trace - + def local_trace(self, frame, event: str, arg): if self.status == 'Kill Scheduled' and event == 'line': print('KILLED ident: %s' % self.ident) self.status = 'Killed' raise SystemExit() return self.local_trace - + def _run_with_trace(self) -> None: """This will replace `threading.Thread`'s `run()` method""" if not self._run: - raise exceptions.ThreadNotInitializedError('Running `_run_with_trace` may cause unintended behaviour, run `start` instead') - + raise exceptions.ThreadNotInitializedError( + 'Running `_run_with_trace` may cause unintended behaviour, run `start` instead' + ) + sys.settrace(self.global_trace) self._run() - @property def result(self) -> _Target_T: """ The return value of the thread - + Raises ------ - ThreadNotInitializedError: If the thread is not intialized + ThreadNotInitializedError: If the thread is not initialized ThreadNotRunningError: If the thread is not running ThreadStillRunningError: If the thread is still running """ @@ -193,26 +193,24 @@ def result(self) -> _Target_T: raise exceptions.ThreadNotInitializedError() if self.status in ['Idle', 'Killed']: raise exceptions.ThreadNotRunningError() - + self._handle_exceptions() if self.status in ['Invoking hooks', 'Completed']: return self._returned_value else: raise exceptions.ThreadStillRunningError() - - + def is_alive(self) -> bool: """ See if thread is still alive Raises ------ - ThreadNotInitializedError: If the thread is not intialized + ThreadNotInitializedError: If the thread is not initialized """ if not self._initialized: raise exceptions.ThreadNotInitializedError() return super().is_alive() - def add_hook(self, hook: HookFunction[_Target_T]) -> None: """ @@ -227,7 +225,6 @@ def add_hook(self, hook: HookFunction[_Target_T]) -> None: """ self.hooks.append(hook) - def join(self, timeout: Optional[float] = None) -> bool: """ Halts the current thread execution until a thread completes or exceeds the timeout @@ -247,14 +244,13 @@ def join(self, timeout: Optional[float] = None) -> bool: """ if not self._initialized: raise exceptions.ThreadNotInitializedError() - + if self.status == ['Idle', 'Killed']: raise exceptions.ThreadNotRunningError() super().join(timeout) self._handle_exceptions() return not self.is_alive() - def get_return_value(self) -> _Target_T: """ @@ -266,7 +262,6 @@ def get_return_value(self) -> _Target_T: """ self.join() return self.result - def kill(self, yielding: bool = False, timeout: float = 5) -> bool: """ @@ -280,7 +275,7 @@ def kill(self, yielding: bool = False, timeout: float = 5) -> bool: Returns ------- :returns bool: False if the it exceeded the timeout - + Raises ------ ThreadNotInitializedError: If the thread is not initialized @@ -288,11 +283,11 @@ def kill(self, yielding: bool = False, timeout: float = 5) -> bool: """ if not self.is_alive(): raise exceptions.ThreadNotRunningError() - + self.status = 'Kill Scheduled' if not yielding: return True - + start = time.perf_counter() while self.status != 'Killed': time.sleep(0.01) @@ -300,28 +295,27 @@ def kill(self, yielding: bool = False, timeout: float = 5) -> bool: return False return True - def start(self) -> None: """ Starts the thread - + Raises ------ - ThreadNotInitializedError: If the thread is not intialized + ThreadNotInitializedError: If the thread is not initialized ThreadStillRunningError: If there already is a running thread """ if self.is_alive(): raise exceptions.ThreadStillRunningError() - + self._run = self.run self.run = self._run_with_trace super().start() +_P = ParamSpec('_P') -_P = ParamSpec('_P') class _ThreadWorker: progress: float thread: Thread @@ -330,6 +324,7 @@ def __init__(self, thread: Thread, progress: float = 0) -> None: self.thread = thread self.progress = progress + class ParallelProcessing(Generic[_Target_P, _Target_T, _Dataset_T]): """ Multi-Threaded Parallel Processing @@ -338,30 +333,29 @@ class ParallelProcessing(Generic[_Target_P, _Target_T, _Dataset_T]): Type-Safe and provides more functionality on top """ - _threads : List[_ThreadWorker] - _completed : int + _threads: List[_ThreadWorker] + _completed: int - status : ThreadStatus - function : TargetFunction - dataset : Sequence[Data_In] - max_threads : int - - overflow_args : Sequence[Overflow_In] + status: ThreadStatus + function: TargetFunction + dataset: Sequence[Data_In] + max_threads: int + + overflow_args: Sequence[Overflow_In] overflow_kwargs: Mapping[str, Overflow_In] - + def __init__( self, function: DatasetFunction[_Dataset_T, _Target_T], dataset: Sequence[_Dataset_T], max_threads: int = 8, - *overflow_args: Overflow_In, - **overflow_kwargs: Overflow_In - ) -> None: + **overflow_kwargs: Overflow_In, + ) -> None: """ Initializes a new Multi-Threaded Pool\n Best for data processing - + Splits a dataset as evenly as it can among the threads and run them in parallel Parameters @@ -389,32 +383,33 @@ def __init__( self.overflow_args = overflow_args self.overflow_kwargs = overflow_kwargs - - def _wrap_function( - self, - function: TargetFunction - ) -> TargetFunction: + def _wrap_function(self, function: TargetFunction) -> TargetFunction: @wraps(function) - def wrapper(index: int, data_chunk: Sequence[_Dataset_T], *args: _Target_P.args, **kwargs: _Target_P.kwargs) -> List[_Target_T]: + def wrapper( + index: int, + data_chunk: Sequence[_Dataset_T], + *args: _Target_P.args, + **kwargs: _Target_P.kwargs, + ) -> List[_Target_T]: computed: List[Data_Out] = [] for i, data_entry in enumerate(data_chunk): v = function(data_entry, *args, **kwargs) computed.append(v) - self._threads[index].progress = round((i+1)/len(data_chunk), 5) + self._threads[index].progress = round((i + 1) / len(data_chunk), 5) self._completed += 1 if self._completed == len(self._threads): self.status = 'Completed' return computed - return wrapper + return wrapper @property def results(self) -> List[_Dataset_T]: """ The return value of the threads if completed - + Raises ------ ThreadNotInitializedError: If the threads are not initialized @@ -428,7 +423,6 @@ def results(self) -> List[_Dataset_T]: for entry in self._threads: results += entry.thread.result return results - def is_alive(self) -> bool: """ @@ -436,12 +430,11 @@ def is_alive(self) -> bool: Raises ------ - ThreadNotInitializedError: If the thread is not intialized + ThreadNotInitializedError: If the thread is not initialized """ if len(self._threads) == 0: raise exceptions.ThreadNotInitializedError() return any(entry.thread.is_alive() for entry in self._threads) - def get_return_values(self) -> List[_Dataset_T]: """ @@ -456,7 +449,6 @@ def get_return_values(self) -> List[_Dataset_T]: entry.thread.join() results += entry.thread.result return results - def join(self) -> bool: """ @@ -473,14 +465,13 @@ def join(self) -> bool: """ if len(self._threads) == 0: raise exceptions.ThreadNotInitializedError() - + if self.status == 'Idle': raise exceptions.ThreadNotRunningError() for entry in self._threads: entry.thread.join() return True - def kill(self) -> None: """ @@ -493,48 +484,48 @@ def kill(self) -> None: """ for entry in self._threads: entry.thread.kill() - def start(self) -> None: """ Starts the threads - + Raises ------ ThreadStillRunningError: If there already is a running thread """ if self.status == 'Running': raise exceptions.ThreadStillRunningError() - + self.status = 'Running' max_threads = min(self.max_threads, len(self.dataset)) parsed_args = self.overflow_kwargs.get('args', []) - name_format = self.overflow_kwargs.get('name') and self.overflow_kwargs['name'] + '%s' - self.overflow_kwargs = { i: v for i,v in self.overflow_kwargs.items() if i != 'name' and i != 'args' } + name_format = ( + self.overflow_kwargs.get('name') and self.overflow_kwargs['name'] + '%s' + ) + self.overflow_kwargs = { + i: v for i, v in self.overflow_kwargs.items() if i != 'name' and i != 'args' + } print(parsed_args, self.overflow_args) for i, data_chunk in enumerate(chunk_split(self.dataset, max_threads)): chunk_thread = Thread( - target = self.function, - args = [i, data_chunk, *parsed_args, *self.overflow_args], - name = name_format and name_format % i or None, - **self.overflow_kwargs + target=self.function, + args=[i, data_chunk, *parsed_args, *self.overflow_args], + name=name_format and name_format % i or None, + **self.overflow_kwargs, ) self._threads.append(_ThreadWorker(chunk_thread, 0)) chunk_thread.start() - - - # Handle abrupt exit def service_shutdown(signum, frame): if Settings.GRACEFUL_EXIT_ENABLED: print('\nCaught signal %d' % signum) print('Gracefully killing active threads') - + for thread in Threads: if isinstance(thread, Thread): try: diff --git a/src/thread/utils/algorithm.py b/src/thread/utils/algorithm.py index 1a20af5..bc531b4 100644 --- a/src/thread/utils/algorithm.py +++ b/src/thread/utils/algorithm.py @@ -14,7 +14,7 @@ def chunk_split(dataset: Sequence[Any], number_of_chunks: int) -> List[List[Any]]: """ Splits a dataset into balanced chunks - + If the size of the dataset is not fully divisible by the number of chunks, it is split like this > `[ [n+1], [n+1], [n+1], [n], [n], [n] ]` @@ -34,7 +34,9 @@ def chunk_split(dataset: Sequence[Any], number_of_chunks: int) -> List[List[Any] AssertionError: The number of chunks specified is larger than the dataset size """ length = len(dataset) - assert length >= number_of_chunks, 'The number of chunks specified is larger than the dataset size' + assert ( + length >= number_of_chunks + ), 'The number of chunks specified is larger than the dataset size' chunk_count = length // number_of_chunks overflow = length % number_of_chunks @@ -50,4 +52,3 @@ def chunk_split(dataset: Sequence[Any], number_of_chunks: int) -> List[List[Any] i = b return split - diff --git a/src/thread/utils/config.py b/src/thread/utils/config.py index 2a89139..f9108c9 100644 --- a/src/thread/utils/config.py +++ b/src/thread/utils/config.py @@ -6,10 +6,8 @@ class Settings: GRACEFUL_EXIT_ENABLED: bool = True - def __init__(self): raise NotImplementedError('This class is not instantiable') - @staticmethod def set_graceful_exit(enabled: bool = True):