From ad9c60931809789123e1d4b191d7f211582fb573 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 8 Jul 2026 22:58:00 -0600 Subject: [PATCH] Fix OSE POD pagination truncating to first 2000 wells The OSE FeatureServer caps a page at maxRecordCount=2000, but chunk_size was 5000. The pagination loop breaks when a page returns fewer rows than chunk_size, so the first full 2000-row page read as "short" and the loop stopped after one request. Only the first 2000 PODs (OBJECTID-ordered = oldest first) were ever fetched statewide, so recent wells were dropped entirely -- leaving the POD-age products near-empty and under-counting every product fed by this source (well correlation, well density). Set chunk_size to 2000 so full pages read as full and the loop pages to the end. Co-Authored-By: Claude Opus 4.8 --- backend/connectors/nmose/source.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 04aef48..895afab 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -19,7 +19,13 @@ class NMOSEPODSiteSource(BaseSiteSource): It is used to fetch site data from the NMOSEPOD API. """ - chunk_size: int = 5000 + # The OSE FeatureServer caps a single page at its maxRecordCount (2000). + # chunk_size must not exceed it: the pagination loop below stops when a page + # comes back smaller than chunk_size, so a chunk_size larger than the server + # cap makes every full 2000-row page look "short" and breaks after page 1 -- + # silently fetching only the first 2000 (oldest, OBJECTID-ordered) PODs and + # dropping every recent well the POD-age products need. + chunk_size: int = 2000 bounding_polygon = NM_STATE_BOUNDING_POLYGON def __init__(self):