From 20d702d3bf9c3200e599271fb2fa6ff17091d670 Mon Sep 17 00:00:00 2001
From: AlexNg <contact@ngjx.org>
Date: Sun, 28 Apr 2024 13:54:01 +0800
Subject: [PATCH 1/3] perf(chunking): Return a generator instead of a list

---
 src/thread/utils/algorithm.py | 11 +++++------
 tests/test_algorithm.py       | 14 ++++++++++----
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/thread/utils/algorithm.py b/src/thread/utils/algorithm.py
index 84c3d9c..0e113b1 100644
--- a/src/thread/utils/algorithm.py
+++ b/src/thread/utils/algorithm.py
@@ -8,10 +8,12 @@
   |_ b.py
 """
 
-from typing import List, Tuple
+from typing import Tuple, Generator
 
 
-def chunk_split(dataset_length: int, number_of_chunks: int) -> List[Tuple[int, int]]:
+def chunk_split(
+    dataset_length: int, number_of_chunks: int
+) -> Generator[Tuple[int, int], None, None]:
     """
     Splits a dataset into balanced chunks
 
@@ -41,13 +43,10 @@ def chunk_split(dataset_length: int, number_of_chunks: int) -> List[Tuple[int, i
     overflow = dataset_length % number_of_chunks
 
     i = 0
-    split = []
     while i < dataset_length:
         chunk_length = chunk_count + int(overflow > 0)
         b = i + chunk_length
 
-        split.append((i, b))
+        yield (i, b)
         overflow -= 1
         i = b
-
-    return split
diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py
index 8d9b4be..6c73038 100644
--- a/tests/test_algorithm.py
+++ b/tests/test_algorithm.py
@@ -1,17 +1,23 @@
 import random
+from typing import Generator
+
 from src.thread.utils import algorithm
 
 
+def test_type():
+    assert isinstance(algorithm.chunk_split(5, 1), Generator)
+
+
 def test_chunking_1():
-    assert algorithm.chunk_split(5, 1) == [(0, 5)]
+    assert list(algorithm.chunk_split(5, 1)) == [(0, 5)]
 
 
 def test_chunking_2():
-    assert algorithm.chunk_split(5, 2) == [(0, 3), (3, 5)]
+    assert list(algorithm.chunk_split(5, 2)) == [(0, 3), (3, 5)]
 
 
 def test_chunking_3():
-    assert algorithm.chunk_split(100, 8) == [
+    assert list(algorithm.chunk_split(100, 8)) == [
         (0, 13),
         (13, 26),
         (26, 39),
@@ -41,5 +47,5 @@ def test_chunking_dynamic():
         i = b
 
     assert (
-        algorithm.chunk_split(dataset_length, thread_count) == heap
+        list(algorithm.chunk_split(dataset_length, thread_count)) == heap
     ), f'\nLength: {dataset_length}\nThreads: {thread_count}\nExpected: {heap}\nActual: {algorithm.chunk_split(dataset_length, thread_count)}'

From 67077ab06bb90cef5790e9b0fcf96a21ebfb3b2e Mon Sep 17 00:00:00 2001
From: AlexNg <contact@ngjx.org>
Date: Sun, 28 Apr 2024 13:56:01 +0800
Subject: [PATCH 2/3] test(perf): Check as we generate instead of writing all
 of chunk to memory

---
 tests/test_algorithm.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py
index 6c73038..4d629cd 100644
--- a/tests/test_algorithm.py
+++ b/tests/test_algorithm.py
@@ -37,15 +37,13 @@ def test_chunking_dynamic():
     expected_chunk_high = dataset_length % thread_count
 
     i = 0
-    heap = []
+    gen = algorithm.chunk_split(dataset_length, thread_count)
     while i < dataset_length:
         chunk_length = expected_chunk_length_low + int(expected_chunk_high > 0)
         b = i + chunk_length
 
-        heap.append((i, b))
+        assert (
+            next(gen) == (i, b)
+        ), f'\nIndex: {i}\nLength: {dataset_length}\nThreads: {thread_count}\nExpected: {(i, b)}\nActual: {next(gen)}'
         expected_chunk_high -= 1
         i = b
-
-    assert (
-        list(algorithm.chunk_split(dataset_length, thread_count)) == heap
-    ), f'\nLength: {dataset_length}\nThreads: {thread_count}\nExpected: {heap}\nActual: {algorithm.chunk_split(dataset_length, thread_count)}'

From 214a6c50a72710f4501cdd03d7ffd0ea4b37cc8f Mon Sep 17 00:00:00 2001
From: AlexNg <contact@ngjx.org>
Date: Sun, 28 Apr 2024 14:06:19 +0800
Subject: [PATCH 3/3] doc: Update return type

---
 src/thread/utils/algorithm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/thread/utils/algorithm.py b/src/thread/utils/algorithm.py
index 0e113b1..c2298dc 100644
--- a/src/thread/utils/algorithm.py
+++ b/src/thread/utils/algorithm.py
@@ -29,7 +29,7 @@ def chunk_split(
 
     Returns
     -------
-    :returns list[tuple[int, int]]: The chunked dataset slices
+    :returns Generator[tuple[int, int], None, None]: The chunked dataset slices
 
     Raises
     ------