From d70695bf6a4b048ebbf54993a0c4e7048986717c Mon Sep 17 00:00:00 2001 From: akash5100 Date: Thu, 1 Sep 2022 21:53:44 +0530 Subject: [PATCH 01/16] merge upstream --- hvpy/utils.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/hvpy/utils.py b/hvpy/utils.py index 0c2a3d3..2030c55 100644 --- a/hvpy/utils.py +++ b/hvpy/utils.py @@ -1,7 +1,9 @@ -from typing import Any, List, Union, Callable +from typing import Any, List, Union, Callable, Optional from pathlib import Path from datetime import datetime +from hvpy.api_groups.movies.queue_movie import queueMovieInputParameters +from hvpy.core import execute_api_call from hvpy.datasource import DataSource from hvpy.event import EventType @@ -150,3 +152,75 @@ def save_file(data: bytearray, filename: Union[Path, str], overwrite: bool = Fal if filename.exists() and not overwrite: raise ValueError(f"{filename} already exists. Use overwrite=True to overwrite.") filename.write_bytes(data) + + +@_add_shared_docstring(queueMovieInputParameters) +def createMovie( + startTime: datetime, + endTime: datetime, + layers: str, + events: str, + eventsLabels: bool, + imageScale: float, + format: Optional[str] = "mp4", + frameRate: Optional[str] = "15", + maxFrames: Optional[str] = None, + scale: Optional[bool] = None, + scaleType: Optional[str] = None, + scaleX: Optional[float] = None, + scaleY: Optional[float] = None, + movieLength: Optional[float] = None, + watermark: Optional[bool] = True, + width: Optional[str] = None, + height: Optional[str] = None, + x0: Optional[str] = None, + y0: Optional[str] = None, + x1: Optional[str] = None, + y1: Optional[str] = None, + x2: Optional[str] = None, + y2: Optional[str] = None, + callback: Optional[str] = None, + size: Optional[int] = None, + movieIcons: Optional[int] = None, + followViewport: Optional[int] = None, + reqObservationDate: Optional[datetime] = None, +): + """ + Automatically creates a movie using the endpoint `queueMovie`, + `getMovieStatus` and `downloadMovie`. + + Parameters + ---------- + {Insert} + """ + params = queueMovieInputParameters( + startTime=startTime, + endTime=endTime, + layers=layers, + events=events, + eventsLabels=eventsLabels, + imageScale=imageScale, + format=format, + frameRate=frameRate, + maxFrames=maxFrames, + scale=scale, + scaleType=scaleType, + scaleX=scaleX, + scaleY=scaleY, + movieLength=movieLength, + watermark=watermark, + width=width, + height=height, + x0=x0, + y0=y0, + x1=x1, + y1=y1, + x2=x2, + y2=y2, + callback=callback, + size=size, + movieIcons=movieIcons, + followViewport=followViewport, + reqObservationDate=reqObservationDate, + ) + res = execute_api_call(input_parameters=params) From f1d3a91f39231b907ef64739338e9e70ff6cd7d2 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Fri, 2 Sep 2022 22:00:19 +0530 Subject: [PATCH 02/16] refactor createMovie function --- hvpy/__init__.py | 3 +- hvpy/helper.py | 111 ++++++++++++++++++++++++++++++++++++++ hvpy/tests/test_helper.py | 18 +++++++ hvpy/utils.py | 76 +------------------------- 4 files changed, 132 insertions(+), 76 deletions(-) create mode 100644 hvpy/helper.py create mode 100644 hvpy/tests/test_helper.py diff --git a/hvpy/__init__.py b/hvpy/__init__.py index 6fe14b1..dc36c65 100644 --- a/hvpy/__init__.py +++ b/hvpy/__init__.py @@ -2,5 +2,6 @@ from .datasource import * from .event import * from .facade import * -from .utils import create_layers, create_events +from .helper import createMovie +from .utils import create_events, create_layers, save_file from .version import __version__ diff --git a/hvpy/helper.py b/hvpy/helper.py new file mode 100644 index 0000000..014105a --- /dev/null +++ b/hvpy/helper.py @@ -0,0 +1,111 @@ +from typing import Union, Optional +from pathlib import Path +from datetime import datetime + +from hvpy import downloadMovie, getMovieStatus +from hvpy.api_groups.movies.queue_movie import queueMovieInputParameters +from hvpy.core import execute_api_call +from hvpy.utils import _add_shared_docstring, save_file + +__all__ = [ + "createMovie", +] + + +@_add_shared_docstring(queueMovieInputParameters) +def createMovie( + startTime: datetime, + endTime: datetime, + layers: str, + events: str, + eventsLabels: bool, + imageScale: float, + format: str = "mp4", + overwrite: bool = False, + filename: Union[str, Path] = None, + frameRate: Optional[str] = "15", + maxFrames: Optional[str] = None, + scale: Optional[bool] = None, + scaleType: Optional[str] = None, + scaleX: Optional[float] = None, + scaleY: Optional[float] = None, + movieLength: Optional[float] = None, + watermark: Optional[bool] = True, + width: Optional[str] = None, + height: Optional[str] = None, + x0: Optional[str] = None, + y0: Optional[str] = None, + x1: Optional[str] = None, + y1: Optional[str] = None, + x2: Optional[str] = None, + y2: Optional[str] = None, + callback: Optional[str] = None, + size: Optional[int] = None, + movieIcons: Optional[int] = None, + followViewport: Optional[int] = None, + reqObservationDate: Optional[datetime] = None, +): + """ + Automatically creates a movie using the endpoint `queueMovie`, + `getMovieStatus` and `downloadMovie`. + + Parameters + ---------- + {Insert} + """ + params = queueMovieInputParameters( + startTime=startTime, + endTime=endTime, + layers=layers, + events=events, + eventsLabels=eventsLabels, + imageScale=imageScale, + format=format, + frameRate=frameRate, + maxFrames=maxFrames, + scale=scale, + scaleType=scaleType, + scaleX=scaleX, + scaleY=scaleY, + movieLength=movieLength, + watermark=watermark, + width=width, + height=height, + x0=x0, + y0=y0, + x1=x1, + y1=y1, + x2=x2, + y2=y2, + callback=callback, + size=size, + movieIcons=movieIcons, + followViewport=followViewport, + reqObservationDate=reqObservationDate, + ) + res = execute_api_call(input_parameters=params) + assert isinstance(res, dict) + + if filename is None: + filename = f"{startTime.isoformat()}_{endTime.isoformat()}" + + while True: + status = getMovieStatus( + id=res["id"], + format=format, + callback=callback, + token=res["token"], + ) + if status["status"] == 2: + break + + binary_data = downloadMovie( + id=res["id"], + format=format, + ) + + save_file( + data=binary_data, + filename=f"{filename}.{format}", + overwrite=overwrite, + ) diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py new file mode 100644 index 0000000..820ff8f --- /dev/null +++ b/hvpy/tests/test_helper.py @@ -0,0 +1,18 @@ +from pathlib import Path + +from hvpy import create_events, create_layers, createMovie +from hvpy.datasource import DataSource + + +def test_create_movie(start_time, end_time, tmp_path): + f1 = tmp_path / "movie" + createMovie( + startTime=start_time, + endTime=end_time, + layers=create_layers([(DataSource.AIA_131, 50)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + filename=f1, + ) + assert Path(f"{f1}.mp4").exists() diff --git a/hvpy/utils.py b/hvpy/utils.py index 2030c55..0c2a3d3 100644 --- a/hvpy/utils.py +++ b/hvpy/utils.py @@ -1,9 +1,7 @@ -from typing import Any, List, Union, Callable, Optional +from typing import Any, List, Union, Callable from pathlib import Path from datetime import datetime -from hvpy.api_groups.movies.queue_movie import queueMovieInputParameters -from hvpy.core import execute_api_call from hvpy.datasource import DataSource from hvpy.event import EventType @@ -152,75 +150,3 @@ def save_file(data: bytearray, filename: Union[Path, str], overwrite: bool = Fal if filename.exists() and not overwrite: raise ValueError(f"{filename} already exists. Use overwrite=True to overwrite.") filename.write_bytes(data) - - -@_add_shared_docstring(queueMovieInputParameters) -def createMovie( - startTime: datetime, - endTime: datetime, - layers: str, - events: str, - eventsLabels: bool, - imageScale: float, - format: Optional[str] = "mp4", - frameRate: Optional[str] = "15", - maxFrames: Optional[str] = None, - scale: Optional[bool] = None, - scaleType: Optional[str] = None, - scaleX: Optional[float] = None, - scaleY: Optional[float] = None, - movieLength: Optional[float] = None, - watermark: Optional[bool] = True, - width: Optional[str] = None, - height: Optional[str] = None, - x0: Optional[str] = None, - y0: Optional[str] = None, - x1: Optional[str] = None, - y1: Optional[str] = None, - x2: Optional[str] = None, - y2: Optional[str] = None, - callback: Optional[str] = None, - size: Optional[int] = None, - movieIcons: Optional[int] = None, - followViewport: Optional[int] = None, - reqObservationDate: Optional[datetime] = None, -): - """ - Automatically creates a movie using the endpoint `queueMovie`, - `getMovieStatus` and `downloadMovie`. - - Parameters - ---------- - {Insert} - """ - params = queueMovieInputParameters( - startTime=startTime, - endTime=endTime, - layers=layers, - events=events, - eventsLabels=eventsLabels, - imageScale=imageScale, - format=format, - frameRate=frameRate, - maxFrames=maxFrames, - scale=scale, - scaleType=scaleType, - scaleX=scaleX, - scaleY=scaleY, - movieLength=movieLength, - watermark=watermark, - width=width, - height=height, - x0=x0, - y0=y0, - x1=x1, - y1=y1, - x2=x2, - y2=y2, - callback=callback, - size=size, - movieIcons=movieIcons, - followViewport=followViewport, - reqObservationDate=reqObservationDate, - ) - res = execute_api_call(input_parameters=params) From f7591bd5c1a662ab2d6eadb71ad2d8b8e0ee8226 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Fri, 2 Sep 2022 22:12:05 +0530 Subject: [PATCH 03/16] add missing params in the docstring --- hvpy/helper.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index 014105a..940fce4 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -20,9 +20,7 @@ def createMovie( events: str, eventsLabels: bool, imageScale: float, - format: str = "mp4", - overwrite: bool = False, - filename: Union[str, Path] = None, + format: Optional[str] = "mp4", frameRate: Optional[str] = "15", maxFrames: Optional[str] = None, scale: Optional[bool] = None, @@ -44,6 +42,8 @@ def createMovie( movieIcons: Optional[int] = None, followViewport: Optional[int] = None, reqObservationDate: Optional[datetime] = None, + overwrite: bool = False, + filename: Union[str, Path] = None, ): """ Automatically creates a movie using the endpoint `queueMovie`, @@ -52,6 +52,11 @@ def createMovie( Parameters ---------- {Insert} + overwrite + Whether to overwrite the file if it already exists. + Default is `False`. + filename + The path to save the file to. """ params = queueMovieInputParameters( startTime=startTime, From 8bf6c8fb1b7b904c9362d136a81348f67a0203e9 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Sat, 3 Sep 2022 21:01:42 +0530 Subject: [PATCH 04/16] Checks for errors and suggested changes --- hvpy/helper.py | 39 +++++++++++++++++++++++++++------------ hvpy/tests/test_helper.py | 8 +++----- hvpy/utils.py | 2 +- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index 940fce4..108dbc7 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -1,10 +1,10 @@ +import time from typing import Union, Optional from pathlib import Path from datetime import datetime -from hvpy import downloadMovie, getMovieStatus +from hvpy import downloadMovie, getMovieStatus, queueMovie from hvpy.api_groups.movies.queue_movie import queueMovieInputParameters -from hvpy.core import execute_api_call from hvpy.utils import _add_shared_docstring, save_file __all__ = [ @@ -44,7 +44,8 @@ def createMovie( reqObservationDate: Optional[datetime] = None, overwrite: bool = False, filename: Union[str, Path] = None, -): + hq: bool = False, +) -> Path: """ Automatically creates a movie using the endpoint `queueMovie`, `getMovieStatus` and `downloadMovie`. @@ -58,7 +59,7 @@ def createMovie( filename The path to save the file to. """ - params = queueMovieInputParameters( + res = queueMovie( startTime=startTime, endTime=endTime, layers=layers, @@ -88,11 +89,13 @@ def createMovie( followViewport=followViewport, reqObservationDate=reqObservationDate, ) - res = execute_api_call(input_parameters=params) assert isinstance(res, dict) - if filename is None: - filename = f"{startTime.isoformat()}_{endTime.isoformat()}" + try: + if res["error"] is not None: + raise RuntimeError(res["error"]) + except KeyError: + pass while True: status = getMovieStatus( @@ -101,16 +104,28 @@ def createMovie( callback=callback, token=res["token"], ) + time.sleep(1) if status["status"] == 2: break binary_data = downloadMovie( id=res["id"], format=format, + hq=hq, ) - save_file( - data=binary_data, - filename=f"{filename}.{format}", - overwrite=overwrite, - ) + if filename is None: + filename = f"{startTime.isoformat()}_{endTime.isoformat()}" + + while True: + try: + save_file( + data=binary_data, + filename=f"{filename}.{format}", + overwrite=overwrite, + ) + break + except FileExistsError: + filename = f"{filename}(1)" + + return Path(f"{filename}.{format}") diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index 820ff8f..7928850 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -1,18 +1,16 @@ -from pathlib import Path - from hvpy import create_events, create_layers, createMovie from hvpy.datasource import DataSource def test_create_movie(start_time, end_time, tmp_path): f1 = tmp_path / "movie" - createMovie( + result = createMovie( startTime=start_time, endTime=end_time, - layers=create_layers([(DataSource.AIA_131, 50)]), + layers=create_layers([(DataSource.AIA_171, 100)]), events=create_events(["AR"]), eventsLabels=True, imageScale=1, filename=f1, ) - assert Path(f"{f1}.mp4").exists() + assert result.exists() diff --git a/hvpy/utils.py b/hvpy/utils.py index 0c2a3d3..2d4d031 100644 --- a/hvpy/utils.py +++ b/hvpy/utils.py @@ -148,5 +148,5 @@ def save_file(data: bytearray, filename: Union[Path, str], overwrite: bool = Fal if isinstance(filename, str): filename = Path(filename) if filename.exists() and not overwrite: - raise ValueError(f"{filename} already exists. Use overwrite=True to overwrite.") + raise FileExistsError(f"{filename} already exists. Use overwrite=True to overwrite.") filename.write_bytes(data) From 263badd750399cabe0f2e981f38b173035b6ff32 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Sat, 3 Sep 2022 22:18:06 +0530 Subject: [PATCH 05/16] fix failing test --- hvpy/tests/test_helper.py | 26 ++++++++++++++++++++++++++ hvpy/tests/test_utils.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index 7928850..7caf603 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -14,3 +14,29 @@ def test_create_movie(start_time, end_time, tmp_path): filename=f1, ) assert result.exists() + assert result == tmp_path / "movie.mp4" + + result = createMovie( + startTime=start_time, + endTime=end_time, + layers=create_layers([(DataSource.AIA_171, 100)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + filename=f1, + ) + assert result.exists() + assert result == tmp_path / "movie(1).mp4" + + result = createMovie( + startTime=start_time, + endTime=end_time, + layers=create_layers([(DataSource.AIA_171, 100)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + filename=f1, + overwrite=True, + ) + assert result.exists() + assert result == tmp_path / "movie.mp4" diff --git a/hvpy/tests/test_utils.py b/hvpy/tests/test_utils.py index 10a48b3..eddc1a0 100644 --- a/hvpy/tests/test_utils.py +++ b/hvpy/tests/test_utils.py @@ -84,7 +84,7 @@ def test_save_file(tmp_path): ) save_file(res, f1, overwrite=False) assert f1.exists() - with pytest.raises(ValueError, match="already exists"): + with pytest.raises(FileExistsError, match="already exists"): save_file(res, f1, overwrite=False) save_file(res, f1, overwrite=True) assert f1.exists() From 0ff4e74a69ae083a5e81a3635c5966c38baeb363 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Sun, 4 Sep 2022 10:35:13 +0530 Subject: [PATCH 06/16] Test to hit the missing lines --- hvpy/tests/test_helper.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index 7caf603..a7d60d6 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -1,3 +1,7 @@ +from datetime import datetime + +import pytest + from hvpy import create_events, create_layers, createMovie from hvpy.datasource import DataSource @@ -40,3 +44,29 @@ def test_create_movie(start_time, end_time, tmp_path): ) assert result.exists() assert result == tmp_path / "movie.mp4" + + +def test_create_movie_with_none_filename(start_time, end_time): + result = createMovie( + startTime=start_time, + endTime=end_time, + layers=create_layers([(DataSource.AIA_171, 100)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + ) + assert result.exists() + + +def test_error_handling(tmp_path): + f1 = tmp_path / "movie" + with pytest.raises(RuntimeError): + createMovie( + startTime=datetime(2010, 1, 1), + endTime=datetime(2010, 1, 2), + layers=create_layers([(DataSource.AIA_171, 100)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + filename=f1, + ) From a6c058996e510525de6d3356d4e71c463e32f135 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Sun, 4 Sep 2022 17:07:46 +0530 Subject: [PATCH 07/16] Remove try-block from func and fix failing py310 test --- hvpy/helper.py | 69 ++++++++++++--------------------------- hvpy/tests/test_helper.py | 17 +++------- 2 files changed, 25 insertions(+), 61 deletions(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index 108dbc7..9a77006 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -37,7 +37,6 @@ def createMovie( y1: Optional[str] = None, x2: Optional[str] = None, y2: Optional[str] = None, - callback: Optional[str] = None, size: Optional[int] = None, movieIcons: Optional[int] = None, followViewport: Optional[int] = None, @@ -58,55 +57,32 @@ def createMovie( Default is `False`. filename The path to save the file to. + Optional, will default to ``f"{starttime}_{endtime}.{format}"``. """ - res = queueMovie( - startTime=startTime, - endTime=endTime, - layers=layers, - events=events, - eventsLabels=eventsLabels, - imageScale=imageScale, - format=format, - frameRate=frameRate, - maxFrames=maxFrames, - scale=scale, - scaleType=scaleType, - scaleX=scaleX, - scaleY=scaleY, - movieLength=movieLength, - watermark=watermark, - width=width, - height=height, - x0=x0, - y0=y0, - x1=x1, - y1=y1, - x2=x2, - y2=y2, - callback=callback, - size=size, - movieIcons=movieIcons, - followViewport=followViewport, - reqObservationDate=reqObservationDate, - ) - assert isinstance(res, dict) + input_params = locals() + input_params.pop("overwrite") + input_params.pop("filename") + input_params.pop("hq") + + res = queueMovie(**input_params) - try: - if res["error"] is not None: - raise RuntimeError(res["error"]) - except KeyError: - pass + if res.get("error"): + raise RuntimeError(res["error"]) while True: status = getMovieStatus( id=res["id"], format=format, - callback=callback, token=res["token"], ) - time.sleep(1) + if status["status"] == 0: + continue + if status["status"] == 1: + time.sleep(5) if status["status"] == 2: break + if status["status"] == 3: + raise RuntimeError(status["error"]) binary_data = downloadMovie( id=res["id"], @@ -117,15 +93,10 @@ def createMovie( if filename is None: filename = f"{startTime.isoformat()}_{endTime.isoformat()}" - while True: - try: - save_file( - data=binary_data, - filename=f"{filename}.{format}", - overwrite=overwrite, - ) - break - except FileExistsError: - filename = f"{filename}(1)" + save_file( + data=binary_data, + filename=f"{filename}.{format}", + overwrite=overwrite, + ) return Path(f"{filename}.{format}") diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index a7d60d6..a40a24e 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -1,3 +1,4 @@ +from pathlib import Path from datetime import datetime import pytest @@ -17,21 +18,10 @@ def test_create_movie(start_time, end_time, tmp_path): imageScale=1, filename=f1, ) + assert isinstance(result, Path) assert result.exists() assert result == tmp_path / "movie.mp4" - result = createMovie( - startTime=start_time, - endTime=end_time, - layers=create_layers([(DataSource.AIA_171, 100)]), - events=create_events(["AR"]), - eventsLabels=True, - imageScale=1, - filename=f1, - ) - assert result.exists() - assert result == tmp_path / "movie(1).mp4" - result = createMovie( startTime=start_time, endTime=end_time, @@ -42,6 +32,7 @@ def test_create_movie(start_time, end_time, tmp_path): filename=f1, overwrite=True, ) + assert isinstance(result, Path) assert result.exists() assert result == tmp_path / "movie.mp4" @@ -55,7 +46,9 @@ def test_create_movie_with_none_filename(start_time, end_time): eventsLabels=True, imageScale=1, ) + assert isinstance(result, Path) assert result.exists() + result.unlink() # clean up def test_error_handling(tmp_path): From 1cd20ac48c282c372de376cdd4c87df4db0855a1 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Sun, 4 Sep 2022 17:47:38 +0530 Subject: [PATCH 08/16] Suggested changes for the filename and format --- hvpy/helper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index 9a77006..45b953e 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -60,6 +60,8 @@ def createMovie( Optional, will default to ``f"{starttime}_{endtime}.{format}"``. """ input_params = locals() + # These are used later on but we want to avoid passing + # them into queueMovie. input_params.pop("overwrite") input_params.pop("filename") input_params.pop("hq") @@ -75,9 +77,7 @@ def createMovie( format=format, token=res["token"], ) - if status["status"] == 0: - continue - if status["status"] == 1: + if status["status"] == 0 or status["status"] == 1: time.sleep(5) if status["status"] == 2: break @@ -91,12 +91,14 @@ def createMovie( ) if filename is None: - filename = f"{startTime.isoformat()}_{endTime.isoformat()}" + filename = f"{startTime.date()}_{endTime.date()}.{format}" + else: + filename = f"{filename}.{format}" save_file( data=binary_data, - filename=f"{filename}.{format}", + filename=filename, overwrite=overwrite, ) - return Path(f"{filename}.{format}") + return Path(filename) From 8af46f2286051961b5f3493054711415262d9a73 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Mon, 5 Sep 2022 17:23:48 +0530 Subject: [PATCH 09/16] Timeout for the infinity loop --- hvpy/helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index 45b953e..c40d272 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -71,16 +71,19 @@ def createMovie( if res.get("error"): raise RuntimeError(res["error"]) + timeout = time.time() + 60 * 5 # 5 minutes while True: status = getMovieStatus( id=res["id"], format=format, token=res["token"], ) - if status["status"] == 0 or status["status"] == 1: + if status["status"] in [0, 1]: time.sleep(5) if status["status"] == 2: break + if time.time() > timeout: + raise RuntimeError("Exceeded timeout of 5 minutes.") if status["status"] == 3: raise RuntimeError(status["error"]) From 54a6aa35b63f624d55d4df83b3f90f7c4d943bd3 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Mon, 5 Sep 2022 18:08:59 +0530 Subject: [PATCH 10/16] timeout params to decide the timeout by the user --- hvpy/helper.py | 14 +++++++++++--- hvpy/tests/test_helper.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index c40d272..9a7054f 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -44,6 +44,7 @@ def createMovie( overwrite: bool = False, filename: Union[str, Path] = None, hq: bool = False, + timeout: float = 5, ) -> Path: """ Automatically creates a movie using the endpoint `queueMovie`, @@ -58,6 +59,12 @@ def createMovie( filename The path to save the file to. Optional, will default to ``f"{starttime}_{endtime}.{format}"``. + hq + Download a higher-quality movie file (valid for "mp4" movies only, ignored otherwise). + Default is `False`, optional. + timeout + The timeout in minutes to wait for the movie to be created. + Default is 5 minutes. """ input_params = locals() # These are used later on but we want to avoid passing @@ -65,13 +72,14 @@ def createMovie( input_params.pop("overwrite") input_params.pop("filename") input_params.pop("hq") + input_params.pop("timeout") res = queueMovie(**input_params) if res.get("error"): raise RuntimeError(res["error"]) - timeout = time.time() + 60 * 5 # 5 minutes + timeout_counter = time.time() + 60 * timeout # Default 5 minutes while True: status = getMovieStatus( id=res["id"], @@ -82,8 +90,8 @@ def createMovie( time.sleep(5) if status["status"] == 2: break - if time.time() > timeout: - raise RuntimeError("Exceeded timeout of 5 minutes.") + if time.time() > timeout_counter: + raise RuntimeError(f"Exceeded timeout of {timeout} minutes.") if status["status"] == 3: raise RuntimeError(status["error"]) diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index a40a24e..71a9fd0 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -51,6 +51,35 @@ def test_create_movie_with_none_filename(start_time, end_time): result.unlink() # clean up +def test_create_movie_timeout(start_time, end_time, tmp_path): + f1 = tmp_path / "movie" + with pytest.raises(RuntimeError): + createMovie( + startTime=start_time, + endTime=end_time, + layers=create_layers([(DataSource.AIA_171, 100)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + filename=f1, + timeout=0.5, + ) + + +def test_error_handling2(tmp_path): + f1 = tmp_path / "movie" + with pytest.raises(RuntimeError): + createMovie( + startTime=datetime(2010, 1, 1), + endTime=datetime(2010, 1, 2), + layers=create_layers([(DataSource.AIA_171, 100)]), + events=create_events(["AR"]), + eventsLabels=True, + imageScale=1, + filename=f1, + ) + + def test_error_handling(tmp_path): f1 = tmp_path / "movie" with pytest.raises(RuntimeError): From e3fc3955b4a31d3e45afb0a87f3d8a03f083fd3a Mon Sep 17 00:00:00 2001 From: akash5100 Date: Mon, 5 Sep 2022 19:29:02 +0530 Subject: [PATCH 11/16] update docstring and change import from top level --- hvpy/helper.py | 6 +++--- hvpy/tests/test_helper.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hvpy/helper.py b/hvpy/helper.py index 9a7054f..0e6c209 100644 --- a/hvpy/helper.py +++ b/hvpy/helper.py @@ -3,8 +3,8 @@ from pathlib import Path from datetime import datetime -from hvpy import downloadMovie, getMovieStatus, queueMovie from hvpy.api_groups.movies.queue_movie import queueMovieInputParameters +from hvpy.facade import downloadMovie, getMovieStatus, queueMovie from hvpy.utils import _add_shared_docstring, save_file __all__ = [ @@ -47,8 +47,8 @@ def createMovie( timeout: float = 5, ) -> Path: """ - Automatically creates a movie using the endpoint `queueMovie`, - `getMovieStatus` and `downloadMovie`. + Automatically creates a movie using `queueMovie`, `getMovieStatus` and + `downloadMovie` functions. Parameters ---------- diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index 71a9fd0..5a7a216 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -3,8 +3,9 @@ import pytest -from hvpy import create_events, create_layers, createMovie from hvpy.datasource import DataSource +from hvpy.helper import createMovie +from hvpy.utils import create_events, create_layers def test_create_movie(start_time, end_time, tmp_path): From 8c2aa59a77a695f1b7e69eaf9a160f134b50f909 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Mon, 5 Sep 2022 15:46:01 +0100 Subject: [PATCH 12/16] various tweaks --- .pre-commit-config.yaml | 4 ++-- MANIFEST.in | 19 ------------------- hvpy/__init__.py | 2 +- hvpy/{helper.py => helpers.py} | 9 +-------- hvpy/tests/test_helper.py | 8 ++++---- 5 files changed, 8 insertions(+), 34 deletions(-) delete mode 100644 MANIFEST.in rename hvpy/{helper.py => helpers.py} (98%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74ff6d3..0f87cdc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,13 +5,13 @@ repos: - id: docformatter args: [--in-place, --pre-summary-newline, --make-summary-multi] - repo: https://github.com/myint/autoflake - rev: v1.5.1 + rev: v1.5.3 hooks: - id: autoflake args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable'] exclude: ".*(.fits|.fts|.fit|.txt|tca.*|extern.*|.rst|.md|__init__.py)$" - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.8.0 hooks: - id: black exclude: ".*(.fits|.fts|.fit|.txt|.csv)$" diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 09ac3e8..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,19 +0,0 @@ -# Exclude specific files -# All files which are tracked by git and not explicitly excluded here are included by setuptools_scm -# Prune folders -prune build -prune docs/_build -prune docs/api -prune .circleci -prune .github -prune .jupyter -prune binder - -# exclude a bunch of common hidden files, you probably want to add your own here -exclude .mailmap -exclude .gitignore -exclude .gitattributes -exclude .editorconfig -exclude .zenodo.json -exclude *.yml -exclude *.yaml diff --git a/hvpy/__init__.py b/hvpy/__init__.py index dc36c65..a86fdde 100644 --- a/hvpy/__init__.py +++ b/hvpy/__init__.py @@ -2,6 +2,6 @@ from .datasource import * from .event import * from .facade import * -from .helper import createMovie +from .helpers import createMovie from .utils import create_events, create_layers, save_file from .version import __version__ diff --git a/hvpy/helper.py b/hvpy/helpers.py similarity index 98% rename from hvpy/helper.py rename to hvpy/helpers.py index 0e6c209..9d0c6ae 100644 --- a/hvpy/helper.py +++ b/hvpy/helpers.py @@ -73,12 +73,9 @@ def createMovie( input_params.pop("filename") input_params.pop("hq") input_params.pop("timeout") - res = queueMovie(**input_params) - if res.get("error"): raise RuntimeError(res["error"]) - timeout_counter = time.time() + 60 * timeout # Default 5 minutes while True: status = getMovieStatus( @@ -87,29 +84,25 @@ def createMovie( token=res["token"], ) if status["status"] in [0, 1]: - time.sleep(5) + time.sleep(3) if status["status"] == 2: break if time.time() > timeout_counter: raise RuntimeError(f"Exceeded timeout of {timeout} minutes.") if status["status"] == 3: raise RuntimeError(status["error"]) - binary_data = downloadMovie( id=res["id"], format=format, hq=hq, ) - if filename is None: filename = f"{startTime.date()}_{endTime.date()}.{format}" else: filename = f"{filename}.{format}" - save_file( data=binary_data, filename=filename, overwrite=overwrite, ) - return Path(filename) diff --git a/hvpy/tests/test_helper.py b/hvpy/tests/test_helper.py index 5a7a216..96a1188 100644 --- a/hvpy/tests/test_helper.py +++ b/hvpy/tests/test_helper.py @@ -4,11 +4,11 @@ import pytest from hvpy.datasource import DataSource -from hvpy.helper import createMovie +from hvpy.helpers import createMovie from hvpy.utils import create_events, create_layers -def test_create_movie(start_time, end_time, tmp_path): +def test_createMovie(start_time, end_time, tmp_path): f1 = tmp_path / "movie" result = createMovie( startTime=start_time, @@ -38,7 +38,7 @@ def test_create_movie(start_time, end_time, tmp_path): assert result == tmp_path / "movie.mp4" -def test_create_movie_with_none_filename(start_time, end_time): +def test_createMovie_with_none_filename(start_time, end_time): result = createMovie( startTime=start_time, endTime=end_time, @@ -52,7 +52,7 @@ def test_create_movie_with_none_filename(start_time, end_time): result.unlink() # clean up -def test_create_movie_timeout(start_time, end_time, tmp_path): +def test_createMovie_timeout(start_time, end_time, tmp_path): f1 = tmp_path / "movie" with pytest.raises(RuntimeError): createMovie( From 4ed4553b0ed9ed1fc190ef32455e85f4e7632534 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Mon, 5 Sep 2022 20:57:45 +0530 Subject: [PATCH 13/16] Adds example in docstring --- hvpy/helpers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/hvpy/helpers.py b/hvpy/helpers.py index 9d0c6ae..ca20af9 100644 --- a/hvpy/helpers.py +++ b/hvpy/helpers.py @@ -65,6 +65,21 @@ def createMovie( timeout The timeout in minutes to wait for the movie to be created. Default is 5 minutes. + + Examples + -------- + >>> from hvpy import createMovie, DataSource, create_events, create_layers + >>> from datetime import datetime, timedelta + >>> createMovie( + ... startTime=datetime.today() - timedelta(days=15, minutes=5), + ... endTime=datetime.today() - timedelta(days=15), + ... layers=create_layers([(DataSource.AIA_171, 100)]), + ... events=create_events(["AR"]), + ... eventsLabels=True, + ... imageScale=1, + ... filename="my_movie", + ... ) + PosixPath('my_movie.mp4') """ input_params = locals() # These are used later on but we want to avoid passing From 2adf633924814c8e799dead7fb9b5c9ebb660892 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Mon, 5 Sep 2022 21:29:43 +0530 Subject: [PATCH 14/16] Fix failing doctest --- hvpy/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hvpy/helpers.py b/hvpy/helpers.py index ca20af9..bc1f9df 100644 --- a/hvpy/helpers.py +++ b/hvpy/helpers.py @@ -79,7 +79,7 @@ def createMovie( ... imageScale=1, ... filename="my_movie", ... ) - PosixPath('my_movie.mp4') + ...Path('my_movie.mp4') """ input_params = locals() # These are used later on but we want to avoid passing From b6722a862742f184e60fe51a3e4acab86647fcd7 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Mon, 5 Sep 2022 21:31:40 +0530 Subject: [PATCH 15/16] tweaks to auto generated file name --- hvpy/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hvpy/helpers.py b/hvpy/helpers.py index bc1f9df..97ebf17 100644 --- a/hvpy/helpers.py +++ b/hvpy/helpers.py @@ -112,7 +112,7 @@ def createMovie( hq=hq, ) if filename is None: - filename = f"{startTime.date()}_{endTime.date()}.{format}" + filename = f"{res['id']}_{startTime.date()}_{endTime.date()}.{format}" else: filename = f"{filename}.{format}" save_file( From ceb22c8dd20cf521bd7baed5505fad070e9890a4 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Mon, 5 Sep 2022 17:20:29 +0100 Subject: [PATCH 16/16] update example --- hvpy/helpers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hvpy/helpers.py b/hvpy/helpers.py index 97ebf17..b04aa48 100644 --- a/hvpy/helpers.py +++ b/hvpy/helpers.py @@ -70,7 +70,7 @@ def createMovie( -------- >>> from hvpy import createMovie, DataSource, create_events, create_layers >>> from datetime import datetime, timedelta - >>> createMovie( + >>> movie_location = createMovie( ... startTime=datetime.today() - timedelta(days=15, minutes=5), ... endTime=datetime.today() - timedelta(days=15), ... layers=create_layers([(DataSource.AIA_171, 100)]), @@ -79,7 +79,10 @@ def createMovie( ... imageScale=1, ... filename="my_movie", ... ) - ...Path('my_movie.mp4') + >>> # This is to cleanup the file created from the example + >>> # you don't need to do this + >>> from pathlib import Path + >>> Path('my_movie.mp4').unlink() """ input_params = locals() # These are used later on but we want to avoid passing