From 60241d7c18b860da0fa326717d6e607c8e271979 Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 12:30:12 -0700 Subject: [PATCH 01/35] CurrentObservations to async --- recordGenerators/CurrentObservations.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index d33c012..1775d88 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -6,6 +6,8 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiofiles +import aiohttp import sys sys.path.append("./py2lib") @@ -34,27 +36,25 @@ for i in MPC.getMetroCities(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(tecci, zipCode): +async def getData(tecci, zipCode): + l.debug('Gathering data for location id ' + tecci) fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/observations/current.xml?language=en-US&units=e&apiKey=' + apiKey + data = "" - #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[67:-30] - l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' + async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as f: + await f.write(i2Doc) + await f.close() - f = open("./.temp/CurrentObservations.i2m", "a") - f.write(i2Doc) - f.close() - -def makeDataFile(): +async def makeDataFile(): l.info("Writing a CurrentObservations record.") header = '' footer = '' From bdbd707b32346e51b1b1b2d2a71ffed91832ef27 Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 12:45:01 -0700 Subject: [PATCH 02/35] Open CurrentObservations.i2m file asynchronously --- recordGenerators/CurrentObservations.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index 1775d88..43bfe57 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -59,25 +59,26 @@ async def makeDataFile(): header = '' footer = '' - with open("./.temp/CurrentObservations.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/CurrentObservations.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(tecciId, zipCodes): getData(x, y) - with open("./.temp/CurrentObservations.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/CurrentObservations.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/CurrentObservations.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/CurrentObservations.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] + # This causes blocking but there's not really much we can do about it :( with open("./.temp/CurrentObservations.i2m", 'rb') as f_in: with gzip.open("./.temp/CurrentObservations.gz", 'wb') as f_out: shutil.copyfileobj(f_in, f_out) From e557ebcacb334500544746e90682b3c7025b13a2 Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 12:52:24 -0700 Subject: [PATCH 03/35] DailyForecast to asynchronous --- recordGenerators/DailyForecast.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index acd7133..4f131e7 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiofiles, aiohttp sys.path.append("./py2lib") sys.path.append("./Util") @@ -32,14 +33,13 @@ for i in MPC.getMetroCities(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(tecci, zipCode): +async def getData(tecci, zipCode): fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/daily/7day.xml?language=en-US&units=e&apiKey=' + apiKey + data = "" - #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[61:-24] @@ -47,32 +47,32 @@ def getData(tecci, zipCode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' - f = open("./.temp/DailyForecast.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/DailyForecast.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): +async def makeDataFile(): l.info("Writing a DailyForecast record.") header = '' footer = '' - with open("./.temp/DailyForecast.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/DailyForecast.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(tecciId, zipCodes): getData(x, y) - with open("./.temp/DailyForecast.i2m", 'a') as end: - end.write(footer) + async with open("./.temp/DailyForecast.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/DailyForecast.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/DailyForecast.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/DailyForecast.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] From 73dbba3a657bb1372febbe4c41b3f904f0ade5f1 Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 12:58:42 -0700 Subject: [PATCH 04/35] getData() not awaited in CO and DF records. --- recordGenerators/CurrentObservations.py | 2 +- recordGenerators/DailyForecast.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index 43bfe57..56e8861 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -63,7 +63,7 @@ async def makeDataFile(): await doc.write(header) for x, y in zip(tecciId, zipCodes): - getData(x, y) + await getData(x, y) async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as end: await end.write(footer) diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index 4f131e7..47fbad6 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -61,7 +61,7 @@ async def makeDataFile(): await doc.write(header) for x, y in zip(tecciId, zipCodes): - getData(x, y) + await getData(x, y) async with open("./.temp/DailyForecast.i2m", 'a') as end: await end.write(footer) From 900cad0b9bdfcf2a32940138d737f33bc518b943 Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 12:59:50 -0700 Subject: [PATCH 05/35] HourlyForecast to asynchronous --- recordGenerators/HourlyForecast.py | 38 ++++++++++++++++-------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index 1e6b097..4bca981 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -5,6 +5,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiofiles, aiohttp import sys sys.path.append("./py2lib") @@ -32,46 +33,47 @@ for i in MPC.getMetroCities(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(tecci, zipCode): +async def getData(tecci, zipCode): + l.debug('Gathering data for location id ' + tecci) fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/hourly/360hour.xml?language=en-US&units=e&apiKey=' + apiKey + data = "" #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[48:-11] - l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' - f = open("./.temp/HourlyForecast.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/HourlyForecast.i2m', 'w') as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): + +async def makeDataFile(): l.info("Writing an HourlyForecast record.") header = '' footer = '' - with open("./.temp/HourlyForecast.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/HourlyForecast.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(tecciId, zipCodes): - getData(x, y) + await getData(x, y) - with open("./.temp/HourlyForecast.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/HourlyForecast.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/HourlyForecast.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/HourlyForecast.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/HourlyForecast.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] From c20b48b6d9db601dae430a0d991d582c54157c0e Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 13:22:25 -0700 Subject: [PATCH 06/35] Make alerts async --- recordGenerators/Alerts.py | 601 +++++++++++++++++++------------------ 1 file changed, 303 insertions(+), 298 deletions(-) diff --git a/recordGenerators/Alerts.py b/recordGenerators/Alerts.py index 25c88f8..55326f6 100644 --- a/recordGenerators/Alerts.py +++ b/recordGenerators/Alerts.py @@ -9,6 +9,7 @@ import xml.dom.minidom import shutil import gzip import logging,coloredlogs +import aiofiles, aiohttp import sys @@ -26,321 +27,325 @@ headlineApiKey = '21d8a80b3d6b444998a80b3d6b1449d3' detailsApiKey = '21d8a80b3d6b444998a80b3d6b1449d3' k = 0 -def getAlerts(location): +async def getAlerts(location): global k fetchUrl = 'https://api.weather.com/v3/alerts/headlines?areaId=' + location + ':US&format=json&language=en-US&apiKey=' + headlineApiKey - response = requests.get(fetchUrl) + # response = requests.get(fetchUrl) - theCode = response.status_code - - #Our global variables - - - #Set the actions based on response code - if theCode == 204: - l.info('No alerts for area ' + location + '.\n') - return - elif theCode == 403: - l.critical("Uh oh! Your API key may not be authorized for alerts. Tsk Tsk. Maybe you shouldn't pirate IBM data :)\n") - return - elif theCode == 401: - l.critical("Uh oh! This request requires authentication. Maybe you shouldn't try to access resources for IBM employee's only :)\n") - return - elif theCode == 404: - l.error("Uh oh! The requested resource cannot be found. This means either the URL is wrong or IBM is having technical difficulties :(\n Or.... They deleted the API :O\n") - return - elif theCode == 405: - l.error("Uh oh! Got a 405! This means that somehow.... someway..... this script made an invalid request. So sad..... So terrible..... :(\n") - return - elif theCode == 406: - l.critical("Uh oh! Got a 406! This means that IBM doesn't like us. :(\n") - return - elif theCode == 408: - l.error("Uh oh! We were too slow in providing IBM our alert request. Although I prefer to say we were Slowly Capable! :)\n") - return - elif theCode == 500: - l.error("Uh oh! Seems IBM's on call IT Tech spilled coffee on the server! Looks like no alerts for a while. Please check back later :)\n") - return - elif theCode == 502 or theCode == 503 or theCode == 504: - l.error("Uh oh! This is why you don't have interns messing with the server configuration. Please stand by while IBM's on call IT Tech resolves the issue :)\n") - return - elif theCode == 200: - pass + # theCode = response.status_code - #Alright lets map our headline variables. - l.debug('Found Alert for ' + location + '\n') - dataH = response.json() - alertsRoot = dataH['alerts'] + theCode = 0 - for x in alertsRoot: - detailKey = x['detailKey'] - #Lets get map our detail variables. - detailsUrl = 'https://api.weather.com/v3/alerts/detail?alertId=' + detailKey + '&format=json&language=en-US&apiKey=' + detailsApiKey - detailsResponse = requests.get(detailsUrl) - dataD = detailsResponse.json() - detailsRoot = dataD['alertDetail'] - theDetailsText = detailsRoot['texts'] - detailsText = theDetailsText[0] - descriptionRaw = detailsText['description'] - language = detailsText['languageCode'] - Identifier = location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC']) - - #Is this for a NWS Zone or County? - last4 = location[2:] - locationType = None - if 'C' in last4: - locationType = 'C' - elif 'Z' in last4: - locationType = 'Z' + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + theCode = await r.status + + #Set the actions based on response code + if theCode == 204: + l.info('No alerts for area ' + location + '.\n') + return + elif theCode == 403: + l.critical("Uh oh! Your API key may not be authorized for alerts. Tsk Tsk. Maybe you shouldn't pirate IBM data :)\n") + return + elif theCode == 401: + l.critical("Uh oh! This request requires authentication. Maybe you shouldn't try to access resources for IBM employee's only :)\n") + return + elif theCode == 404: + l.error("Uh oh! The requested resource cannot be found. This means either the URL is wrong or IBM is having technical difficulties :(\n Or.... They deleted the API :O\n") + return + elif theCode == 405: + l.error("Uh oh! Got a 405! This means that somehow.... someway..... this script made an invalid request. So sad..... So terrible..... :(\n") + return + elif theCode == 406: + l.critical("Uh oh! Got a 406! This means that IBM doesn't like us. :(\n") + return + elif theCode == 408: + l.error("Uh oh! We were too slow in providing IBM our alert request. Although I prefer to say we were Slowly Capable! :)\n") + return + elif theCode == 500: + l.error("Uh oh! Seems IBM's on call IT Tech spilled coffee on the server! Looks like no alerts for a while. Please check back later :)\n") + return + elif theCode == 502 or theCode == 503 or theCode == 504: + l.error("Uh oh! This is why you don't have interns messing with the server configuration. Please stand by while IBM's on call IT Tech resolves the issue :)\n") + return + elif theCode == 200: + pass - #theIdent = str(Identifier) - try: - thecheck = open('./.temp/alertmanifest.txt', "r") - check = thecheck.read() + # Map headline variables + l.debug('Found Alert for ' + location + '\n') + dataH = await r.json() + alertsRoot = dataH['alerts'] + + for x in alertsRoot: + detailKey = x['detailKey'] + #Lets get map our detail variables. + detailsUrl = 'https://api.weather.com/v3/alerts/detail?alertId=' + detailKey + '&format=json&language=en-US&apiKey=' + detailsApiKey + detailsResponse = requests.get(detailsUrl) + dataD = detailsResponse.json() + detailsRoot = dataD['alertDetail'] + theDetailsText = detailsRoot['texts'] + detailsText = theDetailsText[0] + descriptionRaw = detailsText['description'] + language = detailsText['languageCode'] + Identifier = location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC']) - if check.find(Identifier) != -1: - l.debug("Alert already sent...") - return - except FileNotFoundError: - l.warning("alert manifest does not exist (yet)") - k += 1 #We have an alert to send! + #Is this for a NWS Zone or County? + last4 = location[2:] + locationType = None + if 'C' in last4: + locationType = 'C' + elif 'Z' in last4: + locationType = 'Z' + + #theIdent = str(Identifier) + try: + async with aiofiles.open('./.temp/alertmanifest.txt', 'r' ) as checkFile: + c = await checkFile.read() + + if c.find(Identifier) != -1: + l.debug(f"{Identifier} was sent already, skipping..") + return + except FileNotFoundError: + l.warning("alert manifest does not exist (yet)") + + k += 1 #We have an alert to send! + + #Lets Map Our Vocal Codes! + vocalCheck = x['phenomena'] + '_' + x['significance'] + vocalCode = None - #Lets Map Our Vocal Codes! - vocalCheck = x['phenomena'] + '_' + x['significance'] - vocalCode = None - - if vocalCheck == 'HU_W': - vocalCode = 'HE001' - elif vocalCheck == 'TY_W': - vocalCode = 'HE002' - elif vocalCheck == 'HI_W': - vocalCode = 'HE003' - elif vocalCheck == 'TO_A': - vocalCode = 'HE004' - elif vocalCheck == 'SV_A': - vocalCode = 'HE005' - elif vocalCheck == 'HU_A': - vocalCode = 'HE006' - elif vocalCheck == 'TY_A': - vocalCode = 'HE007' - elif vocalCheck == 'TR_W': - vocalCode = 'HE008' - elif vocalCheck == 'TR_A': - vocalCode = 'HE009' - elif vocalCheck == 'TI_W': - vocalCode = 'HE010' - elif vocalCheck == 'HI_A': - vocalCode = 'HE011' - elif vocalCheck == 'TI_A': - vocalCode = 'HE012' - elif vocalCheck == 'BZ_W': - vocalCode = 'HE013' - elif vocalCheck == 'IS_W': - vocalCode = 'HE014' - elif vocalCheck == 'WS_W': - vocalCode = 'HE015' - elif vocalCheck == 'HW_W': - vocalCode = 'HE016' - elif vocalCheck == 'LE_W': - vocalCode = 'HE017' - elif vocalCheck == 'ZR_Y': - vocalCode = 'HE018' - elif vocalCheck == 'CF_W': - vocalCode = 'HE019' - elif vocalCheck == 'LS_W': - vocalCode = 'HE020' - elif vocalCheck == 'WW_Y': - vocalCode = 'HE021' - elif vocalCheck == 'LB_Y': - vocalCode = 'HE022' - elif vocalCheck == 'LE_Y': - vocalCode = 'HE023' - elif vocalCheck == 'BZ_A': - vocalCode = 'HE024' - elif vocalCheck == 'WS_A': - vocalCode = 'HE025' - elif vocalCheck == 'FF_A': - vocalCode = 'HE026' - elif vocalCheck == 'FA_A': - vocalCode = 'HE027' - elif vocalCheck == 'FA_Y': - vocalCode = 'HE028' - elif vocalCheck == 'HW_A': - vocalCode = 'HE029' - elif vocalCheck == 'LE_A': - vocalCode = 'HE030' - elif vocalCheck == 'SU_W': - vocalCode = 'HE031' - elif vocalCheck == 'LS_Y': - vocalCode = 'HE032' - elif vocalCheck == 'CF_A': - vocalCode = 'HE033' - elif vocalCheck == 'ZF_Y': - vocalCode = 'HE034' - elif vocalCheck == 'FG_Y': - vocalCode = 'HE035' - elif vocalCheck == 'SM_Y': - vocalCode = 'HE036' - elif vocalCheck == 'EC_W': - vocalCode = 'HE037' - elif vocalCheck == 'EH_W': - vocalCode = 'HE038' - elif vocalCheck == 'HZ_W': - vocalCode = 'HE039' - elif vocalCheck == 'FZ_W': - vocalCode = 'HE040' - elif vocalCheck == 'HT_Y': - vocalCode = 'HE041' - elif vocalCheck == 'WC_Y': - vocalCode = 'HE042' - elif vocalCheck == 'FR_Y': - vocalCode = 'HE043' - elif vocalCheck == 'EC_A': - vocalCode = 'HE044' - elif vocalCheck == 'EH_A': - vocalCode = 'HE045' - elif vocalCheck == 'HZ_A': - vocalCode = 'HE046' - elif vocalCheck == 'DS_W': - vocalCode = 'HE047' - elif vocalCheck == 'WI_Y': - vocalCode = 'HE048' - elif vocalCheck == 'SU_Y': - vocalCode = 'HE049' - elif vocalCheck == 'AS_Y': - vocalCode = 'HE050' - elif vocalCheck == 'WC_W': - vocalCode = 'HE051' - elif vocalCheck == 'FZ_A': - vocalCode = 'HE052' - elif vocalCheck == 'WC_A': - vocalCode = 'HE053' - elif vocalCheck == 'AF_W': - vocalCode = 'HE054' - elif vocalCheck == 'AF_Y': - vocalCode = 'HE055' - elif vocalCheck == 'DU_Y': - vocalCode = 'HE056' - elif vocalCheck == 'LW_Y': - vocalCode = 'HE057' - elif vocalCheck == 'LS_A': - vocalCode = 'HE058' - elif vocalCheck == 'HF_W': - vocalCode = 'HE059' - elif vocalCheck == 'SR_W': - vocalCode = 'HE060' - elif vocalCheck == 'GL_W': - vocalCode = 'HE061' - elif vocalCheck == 'HF_A': - vocalCode = 'HE062' - elif vocalCheck == 'UP_W': - vocalCode = 'HE063' - elif vocalCheck == 'SE_W': - vocalCode = 'HE064' - elif vocalCheck == 'SR_A': - vocalCode = 'HE065' - elif vocalCheck == 'GL_A': - vocalCode = 'HE066' - elif vocalCheck == 'MF_Y': - vocalCode = 'HE067' - elif vocalCheck == 'MS_Y': - vocalCode = 'HE068' - elif vocalCheck == 'SC_Y': - vocalCode = 'HE069' - elif vocalCheck == 'UP_Y': - vocalCode = 'HE073' - elif vocalCheck == 'LO_Y': - vocalCode = 'HE074' - elif vocalCheck == 'AF_V': - vocalCode = 'HE075' - elif vocalCheck == 'UP_A': - vocalCode = 'HE076' - elif vocalCheck == 'TAV_W': - vocalCode = 'HE077' - elif vocalCheck == 'TAV_A': - vocalCode = 'HE078' - elif vocalCheck == 'TO_W': - vocalCode = 'HE110' - else: - vocalCode = '' + if vocalCheck == 'HU_W': + vocalCode = 'HE001' + elif vocalCheck == 'TY_W': + vocalCode = 'HE002' + elif vocalCheck == 'HI_W': + vocalCode = 'HE003' + elif vocalCheck == 'TO_A': + vocalCode = 'HE004' + elif vocalCheck == 'SV_A': + vocalCode = 'HE005' + elif vocalCheck == 'HU_A': + vocalCode = 'HE006' + elif vocalCheck == 'TY_A': + vocalCode = 'HE007' + elif vocalCheck == 'TR_W': + vocalCode = 'HE008' + elif vocalCheck == 'TR_A': + vocalCode = 'HE009' + elif vocalCheck == 'TI_W': + vocalCode = 'HE010' + elif vocalCheck == 'HI_A': + vocalCode = 'HE011' + elif vocalCheck == 'TI_A': + vocalCode = 'HE012' + elif vocalCheck == 'BZ_W': + vocalCode = 'HE013' + elif vocalCheck == 'IS_W': + vocalCode = 'HE014' + elif vocalCheck == 'WS_W': + vocalCode = 'HE015' + elif vocalCheck == 'HW_W': + vocalCode = 'HE016' + elif vocalCheck == 'LE_W': + vocalCode = 'HE017' + elif vocalCheck == 'ZR_Y': + vocalCode = 'HE018' + elif vocalCheck == 'CF_W': + vocalCode = 'HE019' + elif vocalCheck == 'LS_W': + vocalCode = 'HE020' + elif vocalCheck == 'WW_Y': + vocalCode = 'HE021' + elif vocalCheck == 'LB_Y': + vocalCode = 'HE022' + elif vocalCheck == 'LE_Y': + vocalCode = 'HE023' + elif vocalCheck == 'BZ_A': + vocalCode = 'HE024' + elif vocalCheck == 'WS_A': + vocalCode = 'HE025' + elif vocalCheck == 'FF_A': + vocalCode = 'HE026' + elif vocalCheck == 'FA_A': + vocalCode = 'HE027' + elif vocalCheck == 'FA_Y': + vocalCode = 'HE028' + elif vocalCheck == 'HW_A': + vocalCode = 'HE029' + elif vocalCheck == 'LE_A': + vocalCode = 'HE030' + elif vocalCheck == 'SU_W': + vocalCode = 'HE031' + elif vocalCheck == 'LS_Y': + vocalCode = 'HE032' + elif vocalCheck == 'CF_A': + vocalCode = 'HE033' + elif vocalCheck == 'ZF_Y': + vocalCode = 'HE034' + elif vocalCheck == 'FG_Y': + vocalCode = 'HE035' + elif vocalCheck == 'SM_Y': + vocalCode = 'HE036' + elif vocalCheck == 'EC_W': + vocalCode = 'HE037' + elif vocalCheck == 'EH_W': + vocalCode = 'HE038' + elif vocalCheck == 'HZ_W': + vocalCode = 'HE039' + elif vocalCheck == 'FZ_W': + vocalCode = 'HE040' + elif vocalCheck == 'HT_Y': + vocalCode = 'HE041' + elif vocalCheck == 'WC_Y': + vocalCode = 'HE042' + elif vocalCheck == 'FR_Y': + vocalCode = 'HE043' + elif vocalCheck == 'EC_A': + vocalCode = 'HE044' + elif vocalCheck == 'EH_A': + vocalCode = 'HE045' + elif vocalCheck == 'HZ_A': + vocalCode = 'HE046' + elif vocalCheck == 'DS_W': + vocalCode = 'HE047' + elif vocalCheck == 'WI_Y': + vocalCode = 'HE048' + elif vocalCheck == 'SU_Y': + vocalCode = 'HE049' + elif vocalCheck == 'AS_Y': + vocalCode = 'HE050' + elif vocalCheck == 'WC_W': + vocalCode = 'HE051' + elif vocalCheck == 'FZ_A': + vocalCode = 'HE052' + elif vocalCheck == 'WC_A': + vocalCode = 'HE053' + elif vocalCheck == 'AF_W': + vocalCode = 'HE054' + elif vocalCheck == 'AF_Y': + vocalCode = 'HE055' + elif vocalCheck == 'DU_Y': + vocalCode = 'HE056' + elif vocalCheck == 'LW_Y': + vocalCode = 'HE057' + elif vocalCheck == 'LS_A': + vocalCode = 'HE058' + elif vocalCheck == 'HF_W': + vocalCode = 'HE059' + elif vocalCheck == 'SR_W': + vocalCode = 'HE060' + elif vocalCheck == 'GL_W': + vocalCode = 'HE061' + elif vocalCheck == 'HF_A': + vocalCode = 'HE062' + elif vocalCheck == 'UP_W': + vocalCode = 'HE063' + elif vocalCheck == 'SE_W': + vocalCode = 'HE064' + elif vocalCheck == 'SR_A': + vocalCode = 'HE065' + elif vocalCheck == 'GL_A': + vocalCode = 'HE066' + elif vocalCheck == 'MF_Y': + vocalCode = 'HE067' + elif vocalCheck == 'MS_Y': + vocalCode = 'HE068' + elif vocalCheck == 'SC_Y': + vocalCode = 'HE069' + elif vocalCheck == 'UP_Y': + vocalCode = 'HE073' + elif vocalCheck == 'LO_Y': + vocalCode = 'HE074' + elif vocalCheck == 'AF_V': + vocalCode = 'HE075' + elif vocalCheck == 'UP_A': + vocalCode = 'HE076' + elif vocalCheck == 'TAV_W': + vocalCode = 'HE077' + elif vocalCheck == 'TAV_A': + vocalCode = 'HE078' + elif vocalCheck == 'TO_W': + vocalCode = 'HE110' + else: + vocalCode = '' - #Do some date/time conversions - EndTimeUTCEpoch = x['expireTimeUTC'] - EndTimeUTC = datetime.utcfromtimestamp(EndTimeUTCEpoch).strftime('%Y%m%d%H%M') - #EndTimeUTC = EndTimeUTCString.astimezone(pytz.UTC) + #Do some date/time conversions + EndTimeUTCEpoch = x['expireTimeUTC'] + EndTimeUTC = datetime.utcfromtimestamp(EndTimeUTCEpoch).strftime('%Y%m%d%H%M') + #EndTimeUTC = EndTimeUTCString.astimezone(pytz.UTC) + + expireTimeEpoch = x['expireTimeUTC'] + expireTimeUTC = datetime.utcfromtimestamp(expireTimeEpoch).strftime('%Y%m%d%H%M') + + #V3 Alert API doesn't give us issueTime in UTC. So we have to convert ourselves. Ughhh!! + iTLDTS = x['issueTimeLocal'] + iTLDTO = datetime.strptime(iTLDTS, '%Y-%m-%dT%H:%M:%S%z') + issueTimeToUTC = iTLDTO.astimezone(pytz.UTC) + issueTimeUtc = issueTimeToUTC.strftime('%Y%m%d%H%M') + + processTimeEpoch = x['processTimeUTC'] + processTime = datetime.fromtimestamp(processTimeEpoch).strftime('%Y%m%d%H%M%S') + + #What is the action of this alert? + Action = None + if x['messageType'] == 'Update': + Action = 'CON' + elif x['messageType'] == 'New': + Action = 'NEW' - expireTimeEpoch = x['expireTimeUTC'] - expireTimeUTC = datetime.utcfromtimestamp(expireTimeEpoch).strftime('%Y%m%d%H%M') + #Fix description to replace new lines with space and add XML escape Chars. when needed + + description = ' '.join(descriptionRaw.splitlines()) + description = description.replace('&', '&') + description = description.replace('<', '<') + description = description.replace('>', '>') + description = description.replace('-', '') + description = description.replace(':', '') + + #Is this alert urgent? + urgency ='piss' + if vocalCheck == 'TO_W' or vocalCheck == 'SV_W' or vocalCheck == 'FF_W': + urgency = 'BEUrgent' + else: + urgency = 'BERecord' + + alertMsg = 'NOT_USED' + x['productIdentifier'] + 'NOT_USED' + Action + '' + x['officeCode'] + '' + x['phenomena'] + '' + x['significance'] + '' + x['eventTrackingNumber'] + '' + x['eventDescription'] + 'NOT_USED' + EndTimeUTC + '' + str(x['severityCode']) + 'NOT_USED' + expireTimeUTC + '' + location + '' + x['adminDistrictCode'] + 'NOT_USEDNOT_USEDNOT_USED' + x['identifier'] + '' + processTime + '' + issueTimeUtc + '' + x['headlineText'] + '' + vocalCode + 'NOT_USED' + description + 'NOT_USED' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '' - #V3 Alert API doesn't give us issueTime in UTC. So we have to convert ourselves. Ughhh!! - iTLDTS = x['issueTimeLocal'] - iTLDTO = datetime.strptime(iTLDTS, '%Y-%m-%dT%H:%M:%S%z') - issueTimeToUTC = iTLDTO.astimezone(pytz.UTC) - issueTimeUtc = issueTimeToUTC.strftime('%Y%m%d%H%M') - - processTimeEpoch = x['processTimeUTC'] - processTime = datetime.fromtimestamp(processTimeEpoch).strftime('%Y%m%d%H%M%S') - - #What is the action of this alert? - Action = None - if x['messageType'] == 'Update': - Action = 'CON' - elif x['messageType'] == 'New': - Action = 'NEW' - - #Fix description to replace new lines with space and add XML escape Chars. when needed - - description = ' '.join(descriptionRaw.splitlines()) - description = description.replace('&', '&') - description = description.replace('<', '<') - description = description.replace('>', '>') - description = description.replace('-', '') - description = description.replace(':', '') - - #Is this alert urgent? - urgency ='piss' - if vocalCheck == 'TO_W' or vocalCheck == 'SV_W' or vocalCheck == 'FF_W': - urgency = 'BEUrgent' - else: - urgency = 'BERecord' - - alertMsg = 'NOT_USED' + x['productIdentifier'] + 'NOT_USED' + Action + '' + x['officeCode'] + '' + x['phenomena'] + '' + x['significance'] + '' + x['eventTrackingNumber'] + '' + x['eventDescription'] + 'NOT_USED' + EndTimeUTC + '' + str(x['severityCode']) + 'NOT_USED' + expireTimeUTC + '' + location + '' + x['adminDistrictCode'] + 'NOT_USEDNOT_USEDNOT_USED' + x['identifier'] + '' + processTime + '' + issueTimeUtc + '' + x['headlineText'] + '' + vocalCode + 'NOT_USED' + description + 'NOT_USED' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '' - - #Append BERecord - with open('./.temp/BERecord.xml', "a") as b: - b.write(alertMsg) - b.close() - - #Add our alert to the manifest so we don't keep sending in the same alert every 60 seconds unless an update is issued. - with open('./.temp/alertmanifest.txt', "a") as c: - c.write('\n' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC'])) - c.close() + #Append BERecord + async with aiofiles.open('./.temp/BERecord.xml', "a") as b: + await b.write(alertMsg) + await b.close() + + #Add our alert to the manifest so we don't keep sending in the same alert every 60 seconds unless an update is issued. + async with aiofiles.open('./.temp/alertmanifest.txt', "a") as c: + await c.write('\n' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC'])) + await c.close() -# TODO: This should be converted into a function so it works better with async, that way we're not getting hung up on that time.sleep() call. -def makeRecord(): +async def makeRecord(): global k - with open("./.temp/BERecord.xml", 'a') as BERecord: - BERecord.write('') - BERecord.close() - - for z in alertLocations: - getAlerts(z) - with open('./.temp/BERecord.xml', 'a') as BERecord: - BERecord.write("") - BERecord.close() - - dom = xml.dom.minidom.parse("./.temp/BERecord.xml") - pretty_xml_as_string = dom.toprettyxml(indent = " ") - - with open("./.temp/BERecord.i2m", 'w') as h: - h.write(pretty_xml_as_string[23:]) - h.close() - - # If we don't need to send the i2 an alert, we don't need to gzip it. + # The BERecord XML doesn't need to be written if there's no alerts. if k > 0: + async with aiofiles.open("./.temp/BERecord.xml", 'a') as BERecord: + await BERecord.write('') + await BERecord.close() + + for z in alertLocations: + await getAlerts(z) + + async with aiofiles.open('./.temp/BERecord.xml', 'a') as BERecord: + await BERecord.write("") + await BERecord.close() + + dom = xml.dom.minidom.parse("./.temp/BERecord.xml") + pretty_xml_as_string = dom.toprettyxml(indent = " ") + + async with aiofiles.open("./.temp/BERecord.i2m", 'w') as h: + await h.write(pretty_xml_as_string[23:]) + await h.close() + l.info("Sending alert(s) to the IntelliStar 2!") with open("./.temp/BERecord.i2m", 'rb') as f_in: with gzip.open("./.temp/BERecord.gz", 'wb') as f_out: From b96bbad5d724c4b980f4f7d59dc5d906a667d74a Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 13:25:21 -0700 Subject: [PATCH 07/35] Temporarily remove non-async tasks from main --- main.py | 26 ++++++++++++------------- recordGenerators/CurrentObservations.py | 5 ++++- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/main.py b/main.py index 3520f11..4268849 100644 --- a/main.py +++ b/main.py @@ -38,28 +38,28 @@ l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose") async def grabAlertsLoop(): while True: - Alerts.makeRecord() + await Alerts.makeRecord() await asyncio.sleep(60) async def FiveMinUpdaters(): while True: - CurrentObservations.makeDataFile() + await CurrentObservations.makeDataFile() l.debug("Sleeping for 5 minutes...") await asyncio.sleep(5 * 60) async def HourUpdaters(): while True: - DailyForecast.makeDataFile() - HourlyForecast.makeDataFile() - AirQuality.writeData() - PollenForecast.makeDataFile() - AirportDelays.writeData() - Breathing.makeDataFile() - HeatingAndCooling.makeRecord() - WateringNeeds.makeRecord() - MosquitoActivity.makeRecord() - AchesAndPains.makeRecord() - TideForecast.makeRecord() + await DailyForecast.makeDataFile() + await HourlyForecast.makeDataFile() + # AirQuality.writeData() + # PollenForecast.makeDataFile() + # AirportDelays.writeData() + # Breathing.makeDataFile() + # HeatingAndCooling.makeRecord() + # WateringNeeds.makeRecord() + # MosquitoActivity.makeRecord() + # AchesAndPains.makeRecord() + # TideForecast.makeRecord() l.debug("Sleeping for an hour...") await asyncio.sleep(60 * 60) diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index 56e8861..f0ca993 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -78,7 +78,10 @@ async def makeDataFile(): files = [] commands = [] - # This causes blocking but there's not really much we can do about it :( + """ + TODO: This can be ran in a seperate thread using loop.run_in_executor() according to the python discord. + ! This should probably be implemented ASAP. + """ with open("./.temp/CurrentObservations.i2m", 'rb') as f_in: with gzip.open("./.temp/CurrentObservations.gz", 'wb') as f_out: shutil.copyfileobj(f_in, f_out) From f66e45ef01c28d40eaa73d9291e8c77d2e7ef86d Mon Sep 17 00:00:00 2001 From: April Date: Sat, 12 Nov 2022 13:30:10 -0700 Subject: [PATCH 08/35] Fix FileNotFound error in Alerts --- recordGenerators/Alerts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recordGenerators/Alerts.py b/recordGenerators/Alerts.py index 55326f6..2a00f63 100644 --- a/recordGenerators/Alerts.py +++ b/recordGenerators/Alerts.py @@ -359,7 +359,7 @@ async def makeRecord(): bit.sendFile(files, commands, 1, 0) os.remove(gZipFile) k = 0 - - os.remove("./.temp/BERecord.xml") - os.remove("./.temp/BERecord.i2m") + + os.remove("./.temp/BERecord.xml") + os.remove("./.temp/BERecord.i2m") \ No newline at end of file From 9cc4633e8f6c93decde521bc6b6962e929101159 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 14:29:16 -0700 Subject: [PATCH 09/35] __init__ file for record generators --- recordGenerators/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 recordGenerators/__init__.py diff --git a/recordGenerators/__init__.py b/recordGenerators/__init__.py new file mode 100644 index 0000000..e69de29 From 38dc9c005cfc102166d61f70f8764186ca72212b Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 14:29:46 -0700 Subject: [PATCH 10/35] Make each record update its own task --- RecordTasks.py | 22 +++++++++++++++++ main.py | 65 +++++++++----------------------------------------- 2 files changed, 33 insertions(+), 54 deletions(-) create mode 100644 RecordTasks.py diff --git a/RecordTasks.py b/RecordTasks.py new file mode 100644 index 0000000..4d7be17 --- /dev/null +++ b/RecordTasks.py @@ -0,0 +1,22 @@ +import asyncio +from recordGenerators import CurrentObservations,HourlyForecast,DailyForecast + + +""" This houses the tasks needed to update the data records concurrently + I have no idea if this is a messy way to do things, but it will be worked upon if it is. +""" + +async def coTask(): + while True: + await CurrentObservations.makeDataFile() + asyncio.sleep(5 * 60) + +async def hfTask(): + while True: + await HourlyForecast.makeDataFile() + asyncio.sleep(60 * 60) + +async def dfTask(): + while True: + await DailyForecast.makeDataFile() + asyncio.sleep(60 * 60) \ No newline at end of file diff --git a/main.py b/main.py index 4268849..80781d8 100644 --- a/main.py +++ b/main.py @@ -2,10 +2,10 @@ import asyncio from asyncio.log import logger from asyncore import loop import logging,coloredlogs -from recordGenerators import Alerts,CurrentObservations,DailyForecast,HourlyForecast,AirportDelays,AirQuality,HeatingAndCooling,PollenForecast,Breathing, AchesAndPains, MosquitoActivity, WateringNeeds, TideForecast from radar import TWCRadarCollector import os from datetime import datetime +import RecordTasks l = logging.getLogger(__name__) coloredlogs.install(logger=l) @@ -36,58 +36,15 @@ Alerts: 5 minutes l.info("Starting i2RecordCollector") l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose") -async def grabAlertsLoop(): - while True: - await Alerts.makeRecord() - await asyncio.sleep(60) +async def main(): + coTask = asyncio.create_task(RecordTasks.coTask()) + hfTask = asyncio.create_task(RecordTasks.hfTask()) + dfTask = asyncio.create_task(RecordTasks.dfTask()) -async def FiveMinUpdaters(): - while True: - await CurrentObservations.makeDataFile() - l.debug("Sleeping for 5 minutes...") - await asyncio.sleep(5 * 60) + # In theory, these should all run concurrently without problems + await coTask + await hfTask + await dfTask -async def HourUpdaters(): - while True: - await DailyForecast.makeDataFile() - await HourlyForecast.makeDataFile() - # AirQuality.writeData() - # PollenForecast.makeDataFile() - # AirportDelays.writeData() - # Breathing.makeDataFile() - # HeatingAndCooling.makeRecord() - # WateringNeeds.makeRecord() - # MosquitoActivity.makeRecord() - # AchesAndPains.makeRecord() - # TideForecast.makeRecord() - l.debug("Sleeping for an hour...") - await asyncio.sleep(60 * 60) - -async def radarCollector(): - mosaicUpdateIntervals = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] - satradUpdateIntervals = [0, 10, 20, 30, 40, 50] - - while True: - # Server takes ~15 - 35 seconds on average to fully generate a frame, use 40 seconds - # to make sure the radar frame is fully good to go - if datetime.now().minute in mosaicUpdateIntervals and datetime.now().second == 40: - await TWCRadarCollector.collect("radarmosaic") - - if datetime.now().minute in satradUpdateIntervals and datetime.now().second == 45: - await TWCRadarCollector.collect("satrad") - - await asyncio.sleep(1) - -loop = asyncio.get_event_loop() -alertTask = loop.create_task(grabAlertsLoop()) -CCtask = loop.create_task(FiveMinUpdaters()) -ForecastsTask = loop.create_task(HourUpdaters()) - -if useRadarServer: radarTask = loop.create_task(radarCollector()) - -try: - loop.run_until_complete(alertTask) - loop.run_until_complete(CCtask) - loop.run_until_complete(ForecastsTask) - if useRadarServer: loop.run_until_complete(radarTask) -except asyncio.CancelledError: pass \ No newline at end of file +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From 204520753ac1c9492cf7ea7be710536dbc7e9742 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 14:31:42 -0700 Subject: [PATCH 11/35] await asyncio.sleep() calls in RecordTasks --- RecordTasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RecordTasks.py b/RecordTasks.py index 4d7be17..93d62c0 100644 --- a/RecordTasks.py +++ b/RecordTasks.py @@ -9,14 +9,14 @@ from recordGenerators import CurrentObservations,HourlyForecast,DailyForecast async def coTask(): while True: await CurrentObservations.makeDataFile() - asyncio.sleep(5 * 60) + await asyncio.sleep(5 * 60) async def hfTask(): while True: await HourlyForecast.makeDataFile() - asyncio.sleep(60 * 60) + await asyncio.sleep(60 * 60) async def dfTask(): while True: await DailyForecast.makeDataFile() - asyncio.sleep(60 * 60) \ No newline at end of file + await asyncio.sleep(60 * 60) \ No newline at end of file From ac2835e87e6afb00f8ab77bddd1aa9b8df9e4d1c Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 14:33:44 -0700 Subject: [PATCH 12/35] Use aiofiles instead of stock open() --- recordGenerators/DailyForecast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index 47fbad6..9950f69 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -63,7 +63,7 @@ async def makeDataFile(): for x, y in zip(tecciId, zipCodes): await getData(x, y) - async with open("./.temp/DailyForecast.i2m", 'a') as end: + async with aiofiles.open("./.temp/DailyForecast.i2m", 'a') as end: await end.write(footer) From 101b0545f3ad396dc88bd0fc01e5cd4f4237e966 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:12:41 -0700 Subject: [PATCH 13/35] Alerts collector task --- RecordTasks.py | 7 ++++++- main.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/RecordTasks.py b/RecordTasks.py index 93d62c0..d1cf5aa 100644 --- a/RecordTasks.py +++ b/RecordTasks.py @@ -1,11 +1,16 @@ import asyncio -from recordGenerators import CurrentObservations,HourlyForecast,DailyForecast +from recordGenerators import Alerts,CurrentObservations,HourlyForecast,DailyForecast """ This houses the tasks needed to update the data records concurrently I have no idea if this is a messy way to do things, but it will be worked upon if it is. """ +async def alertsTask(): + while True: + await Alerts.makeRecord() + await asyncio.sleep(60) + async def coTask(): while True: await CurrentObservations.makeDataFile() diff --git a/main.py b/main.py index 80781d8..329d58d 100644 --- a/main.py +++ b/main.py @@ -37,11 +37,13 @@ l.info("Starting i2RecordCollector") l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose") async def main(): + alertsTask = asyncio.create_task(RecordTasks.alertsTask()) coTask = asyncio.create_task(RecordTasks.coTask()) hfTask = asyncio.create_task(RecordTasks.hfTask()) dfTask = asyncio.create_task(RecordTasks.dfTask()) # In theory, these should all run concurrently without problems + await alertsTask await coTask await hfTask await dfTask From b14bafa26d6cdd2cdecf244ffdfcdbf75a067162 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:16:42 -0700 Subject: [PATCH 14/35] Comment unused code in starbundle.py --- py2Lib/starbundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py2Lib/starbundle.py b/py2Lib/starbundle.py index ae00c95..306cfd8 100644 --- a/py2Lib/starbundle.py +++ b/py2Lib/starbundle.py @@ -42,4 +42,4 @@ def makeStarBundle(Directory, Type, flag, Version, date, sendAfter): #Directory which contains Files to be bundled Type Flags Version Date SendImmediately(Does not apply to this script) -makeStarBundle('./.temp/i2State/SD/Managed/Events', 'Managed', 'Domestic_SD_Universe', '637898877227230030', '09/28/2022', 0) \ No newline at end of file +# makeStarBundle('./.temp/i2State/SD/Managed/Events', 'Managed', 'Domestic_SD_Universe', '637898877227230030', '09/28/2022', 0) \ No newline at end of file From 95ff8dec904ba289796c4504c14a555290553e55 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:17:08 -0700 Subject: [PATCH 15/35] Get rid of debug code in the LFRecord handler --- records/LFRecord.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/records/LFRecord.py b/records/LFRecord.py index 330b3b0..cd15b95 100644 --- a/records/LFRecord.py +++ b/records/LFRecord.py @@ -37,8 +37,4 @@ def getLatLong(locId: str): return fetched[0] + "/" + fetched[1] def getLocationInfo(locId: str): - pass - -print(getCoopId('USAZ0278')) -print(getZip('USAZ0278')) -print(getLatLong('USAZ0278')) \ No newline at end of file + pass \ No newline at end of file From 0f003fee917563f0fb41886f311dbfe3ec87d141 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:27:14 -0700 Subject: [PATCH 16/35] Make first-time run file creation asynchronous --- main.py | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/main.py b/main.py index 329d58d..2df865a 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,4 @@ -import asyncio +import asyncio, aiofiles from asyncio.log import logger from asyncore import loop import logging,coloredlogs @@ -12,31 +12,36 @@ coloredlogs.install(logger=l) useRadarServer = True -# Create dirs and files -if not os.path.exists('.temp/'): - os.makedirs('.temp/') - -if not os.path.exists('.temp/tiles/'): - os.makedirs('.temp/tiles/') - -if not os.path.exists('.temp/tiles/output/'): - os.makedirs('.temp/tiles/output/') - -if not os.path.exists('.temp/msgId.txt'): - print("Creating initial msgId file") - with open('.temp/msgId.txt', "w") as f: - f.write("410080515") - - -""" -CurrentConditions: Every 5 minutes -Daily Forecasts, Hourlies, etc: 60 minutes -Alerts: 5 minutes -""" l.info("Starting i2RecordCollector") l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose") +async def createTemp(): + """ Used on a first time run, creates necessary files & directories for the message encoder to work properly. """ + if not (os.path.exists('./.temp/')): + l.info("Creating necessary directories & files..") + os.mkdir('./.temp') + + # Used for the record generator + os.mkdir('./.temp/tiles/') + os.mkdir('./.temp/tiles/output/') + + # Used for radar server downloads + os.mkdir('./.temp/output') + os.mkdir('./.temp/output/radarmosaic') + os.mkdir('./.temp/output/satrad') + + # Create msgId file for bit.py + async with aiofiles.open('./.temp/msgId.txt', 'w') as msgId: + await msgId.write('410080515') + await msgId.close() + else: + l.debug(".temp file exists") + return + + async def main(): + await createTemp() + alertsTask = asyncio.create_task(RecordTasks.alertsTask()) coTask = asyncio.create_task(RecordTasks.coTask()) hfTask = asyncio.create_task(RecordTasks.hfTask()) From cfca3d718af3685dfd4fa6a57323aa702f8d8414 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:32:14 -0700 Subject: [PATCH 17/35] Potential fix for georef on the RadarProcessor --- radar/RadarProcessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radar/RadarProcessor.py b/radar/RadarProcessor.py index 9e1967b..65d62bf 100644 --- a/radar/RadarProcessor.py +++ b/radar/RadarProcessor.py @@ -55,8 +55,8 @@ def WorldCoordinateToTile(coord: Point) -> Point: scale = 1 << 6 return Point( - x = math.floor(coord.x * scale / 256), - y = math.floor(coord.y * scale / 256) + x = math.floor(coord.x * scale / 255), + y = math.floor(coord.y * scale / 255) ) def WorldCoordinateToPixel(coord: Point) -> Point: @@ -68,10 +68,10 @@ def WorldCoordinateToPixel(coord: Point) -> Point: ) def LatLongProject(lat, long) -> Point: - siny = math.sin(lat * math.pi / 185) + siny = math.sin(lat * math.pi / 180) siny = min(max(siny, -0.9999), 0.9999) return Point( x = 256 * (0.5 + long / 360), y = 256 * (0.5 - math.log((1 + siny) / (1 - siny)) / (4 * math.pi)) - ) + ) \ No newline at end of file From 47fe9a9c47a0cb0d826b6be40d25080bdd76d2cc Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:41:01 -0700 Subject: [PATCH 18/35] Add national airport codes to getAirportCodes() --- Util/MachineProductCfg.py | 85 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/Util/MachineProductCfg.py b/Util/MachineProductCfg.py index b6d4970..6549eb2 100644 --- a/Util/MachineProductCfg.py +++ b/Util/MachineProductCfg.py @@ -50,9 +50,90 @@ def getTideStations(): def getAirportCodes(): """ Returns all of the airport identifiers present in the MachineProductCfg """ - airports = [] + airports = [ + 'ATL', + 'LAX', + 'ORD', + 'DFW', + 'JFK', + 'DEN', + 'SFO', + 'CLT', + 'LAS', + 'PHX', + 'IAH', + 'MIA', + 'SEA', + 'EWR', + 'MCO', + 'MSP', + 'DTW', + 'BOS', + 'PHL', + 'LGA', + 'FLL', + 'BWI', + 'IAD', + 'MDW', + 'SLC', + 'DCA', + 'HNL', + 'SAN', + 'TPA', + 'PDX', + 'STL', + 'HOU', + 'BNA', + 'AUS', + 'OAK', + 'MSY', + 'RDU', + 'SJC', + 'SNA', + 'DAL', + 'SMF', + 'SAT', + 'RSW', + 'PIT', + 'CLE', + 'IND', + 'MKE', + 'CMH', + 'OGG', + 'PBI', + 'BDL', + 'CVG', + 'JAX', + 'ANC', + 'BUF', + 'ABQ', + 'ONT', + 'OMA', + 'BUR', + 'OKC', + 'MEM', + 'PVD', + 'RIC', + 'SDF', + 'RNO', + 'TUS', + 'CHS', + 'ORF', + 'PWM', + 'GRR', + 'BHM', + 'LIT', + 'DSM', + 'FAR', + 'FSD', + 'ICT', + 'LBB', + 'BIL', + 'BOI', + 'GEG' + ] for i in data['Config']['ConfigDef']['ConfigItems']['ConfigItem']: - if "Airport" in i['@key'] and i['@value'] != "": + if "Airport" in i['@key'] and i['@value'] != "" and not i['@value'] in airports: # Split the string up airports.append(i['@value'].split("_")[2]) From a9aa0174d451a1a9e45bae7da2f3ba0b7c6c98fa Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:46:55 -0700 Subject: [PATCH 19/35] HeatingAndCooling to asynchronous --- recordGenerators/HeatingAndCooling.py | 45 ++++++++++++++------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/recordGenerators/HeatingAndCooling.py b/recordGenerators/HeatingAndCooling.py index 214fff7..787739e 100644 --- a/recordGenerators/HeatingAndCooling.py +++ b/recordGenerators/HeatingAndCooling.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,48 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/heatCool/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" - - res = requests.get(fetchUrl) - - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write HeatingAndCooling record -- Status code {res.status_code}") - return + data = "" - data = res.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write HeatingAndCooling record -- Status code {r.status}") + return + + data = await r.text() + + # data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/HeatingAndCooling.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): l.info("Writing HeatingAndCooling record.") header = '' footer = '' - with open('./.temp/HeatingAndCooling.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/HeatingAndCooling.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/HeatingAndCooling.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/HeatingAndCooling.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip From e2554e11b9d1f11f99e0ad9a63d926ab3bf5a34b Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:49:25 -0700 Subject: [PATCH 20/35] Breathing to asynchronous --- recordGenerators/Breathing.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/recordGenerators/Breathing.py b/recordGenerators/Breathing.py index 6d93690..37e79b9 100644 --- a/recordGenerators/Breathing.py +++ b/recordGenerators/Breathing.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles sys.path.append("./py2lib") sys.path.append("./Util") @@ -30,14 +31,14 @@ l.debug(coopIds, geocodes) apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/breathing/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" + data = "" #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[63:-26] @@ -45,32 +46,32 @@ def getData(coopId, geocode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(coopId) + '' - f = open("./.temp/Breathing.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/Breathing.i2m", "a") as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): +async def makeDataFile(): l.info("Writing a Breathing forecast record.") header = '' footer = '' - with open("./.temp/Breathing.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/Breathing.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(coopIds, geocodes): - getData(x, y) + await getData(x, y) - with open("./.temp/Breathing.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/Breathing.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/Breathing.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/Breathing.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/Breathing.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] From 1f116850ce7c43cecb00bbc7dd547979dff1bc5a Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:52:15 -0700 Subject: [PATCH 21/35] AirQuality to asynchronous --- recordGenerators/AirQuality.py | 35 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/recordGenerators/AirQuality.py b/recordGenerators/AirQuality.py index cb09528..8334783 100644 --- a/recordGenerators/AirQuality.py +++ b/recordGenerators/AirQuality.py @@ -4,6 +4,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles l = logging.getLogger(__name__) coloredlogs.install() @@ -27,22 +28,24 @@ for i in MPC.getPrimaryLocations(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(epaId, zipcode): +async def getData(epaId, zipcode): url = f"https://api.weather.com/v1/location/{zipcode}:4:US/airquality.xml?language=en-US&apiKey={apiKey}" + data = "" - res = requests.get(url=url) - - data = res.text + async with aiohttp.ClientSession() as s: + async with s.get(url) as r: + data = await r.text() + newData = data[57:-11] # Write to i2doc file i2Doc = f'' + '' + newData + f'{epaId}' - f = open("./.temp/AirQuality.i2m", 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/AirQuality.i2m", 'a') as f: + await f.write(i2Doc) + await f.close() -def writeData(): +async def writeData(): useData = False workingEpaIds = [] @@ -62,21 +65,21 @@ def writeData(): header = '' footer = "" - with open("./.temp/AirQuality.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/AirQuality.i2m", 'w') as doc: + await doc.write(header) for (x, y) in zip(workingEpaIds, zipCodes): - getData(x, y) + await getData(x, y) - with open("./.temp/AirQuality.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/AirQuality.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/AirQuality.i2m") xmlPretty = dom.toprettyxml(indent = " ") - with open("./.temp/AirQuality.i2m", 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open("./.temp/AirQuality.i2m", 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() files = [] commands = [] From 704c33201fa95886331a754d8d08e32c683d0379 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 15:56:01 -0700 Subject: [PATCH 22/35] AirportDelays to asynchronous --- recordGenerators/AirportDelays.py | 49 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index 4edae1c..7da8b04 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -4,6 +4,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles import sys sys.path.append("./py2lib") @@ -29,56 +30,56 @@ l.debug(airports) apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(airport): +async def getData(airport): url = f"https://api.weather.com/v1/airportcode/{airport}/airport/delays.xml?language=en-US&apiKey={apiKey}" + data = "" - res = requests.get(url=url) + async with aiohttp.ClientSession() as s: + async with s.get(url) as r: + data = await r.text() - data = res.text newData = data[48:-11] # Write to i2doc file i2Doc = f'' + '' + newData + f'{airport}' - f = open("./.temp/AirportDelays.i2m", 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/AirportDelays.i2m", 'a') as f: + await f.write(i2Doc) + await f.close() -def writeData(): +async def writeData(): useData = False airportsWithDelays = [] for x in airports: - # Do a quick check to see if the airport in question has a delay or not - res = requests.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") - - if (res.status_code != 200): - l.debug(f"[AIRPORT DELAYS] No delays for {x} found, skipping..") - else: - airportsWithDelays.append(x) - l.debug(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..") - useData = True + async with aiohttp.ClientSession() as s: + async with s.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") as r: + if r.status != 200: + l.debug(f"No delay for {x} found, skipping..") + else: + airportsWithDelays.append(x) + useData = True if (useData): l.info("Writing an AirportDelays record.") header = '' footer = "" - with open("./.temp/AirportDelays.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/AirportDelays.i2m", 'w') as doc: + await doc.write(header) for x in airportsWithDelays: - getData(x) + await getData(x) - with open("./.temp/AirportDelays.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/AirportDelays.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/AirportDelays.i2m") prettyXml = dom.toprettyxml(indent=" ") - with open("./.temp/AirportDelays.i2m", 'w') as g: - g.write(prettyXml) - g.close() + async with aiofiles.open("./.temp/AirportDelays.i2m", 'w') as g: + await g.write(prettyXml) + await g.close() files = [] commands = [] From f1c633febc296906fdc82b1bd7d5f551c685d7ee Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:00:38 -0700 Subject: [PATCH 23/35] AchesAndPains to asynchronous --- recordGenerators/AchesAndPains.py | 43 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/recordGenerators/AchesAndPains.py b/recordGenerators/AchesAndPains.py index 6631ca1..e434125 100644 --- a/recordGenerators/AchesAndPains.py +++ b/recordGenerators/AchesAndPains.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,48 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/achePain/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" + data = "" + + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write AchesAndPains record -- status code {r.status}") + return + + data = await r.text() - res = requests.get(fetchUrl) - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write AchesAndPains record -- Status code {res.status_code}") - return - - data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/AchesAndPains.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): l.info("Writing AchesAndPains record.") header = '' footer = '' - with open('./.temp/AchesAndPains.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/AchesAndPains.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/AchesAndPains.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/AchesAndPains.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip From f18b52bcf56d10c04fba58b01f6db404fe79f732 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:03:08 -0700 Subject: [PATCH 24/35] MosquitoActivity to asynchronous --- recordGenerators/MosquitoActivity.py | 43 +++++++++++++++------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/recordGenerators/MosquitoActivity.py b/recordGenerators/MosquitoActivity.py index 0486dc3..cb22f47 100644 --- a/recordGenerators/MosquitoActivity.py +++ b/recordGenerators/MosquitoActivity.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,48 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/mosquito/daily/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" + data = "" + + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write MosquitoActivity record -- status code {r.status}") + return + + data = await r.text() - res = requests.get(fetchUrl) - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write MosquitoActivity record -- Status code {res.status_code}") - return - - data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/MosquitoActivity.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): l.info("Writing MosquitoActivity record.") header = '' footer = '' - with open('./.temp/MosquitoActivity.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/MosquitoActivity.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/MosquitoActivity.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/MosquitoActivity.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip From 68140eb0b2d89cbfdbfe348417ecc384c8b746d5 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:05:06 -0700 Subject: [PATCH 25/35] PollenForecast to asynchronous --- recordGenerators/PollenForecast.py | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/recordGenerators/PollenForecast.py b/recordGenerators/PollenForecast.py index 11a8eed..7c3f475 100644 --- a/recordGenerators/PollenForecast.py +++ b/recordGenerators/PollenForecast.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging, coloredlogs +import aiohttp, aiofiles sys.path.append("./py2lib") sys.path.append("./Util") @@ -31,14 +32,13 @@ l.debug(pollenIds, geocodes) apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(pollenId, geocode): +async def getData(pollenId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/pollen/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" - + data = "" #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[63:-26] @@ -46,32 +46,32 @@ def getData(pollenId, geocode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(pollenId) + '' - f = open("./.temp/PollenForecast.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/PollenForecast.i2m", "a") as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): +async def makeDataFile(): l.info("Writing a PollenForecast record.") header = '' footer = '' - with open("./.temp/PollenForecast.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/PollenForecast.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(pollenIds, geocodes): - getData(x, y) + await getData(x, y) - with open("./.temp/PollenForecast.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/PollenForecast.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/PollenForecast.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/PollenForecast.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/PollenForecast.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] From 639db76723fe82d489f0b61506802331970854bf Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:08:10 -0700 Subject: [PATCH 26/35] TideForecast to asynchronous --- recordGenerators/TideForecast.py | 45 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/recordGenerators/TideForecast.py b/recordGenerators/TideForecast.py index 72d535a..bb69863 100644 --- a/recordGenerators/TideForecast.py +++ b/recordGenerators/TideForecast.py @@ -1,6 +1,4 @@ import shutil -from xmlrpc.client import DateTime -import requests import logging,coloredlogs import datetime from py2Lib import bit @@ -9,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles l = logging.getLogger(__name__) coloredlogs.install() @@ -22,31 +21,33 @@ for i in MPC.getTideStations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(tideStation, geocode): +async def getData(tideStation, geocode): today = datetime.date.today() startDate = today.strftime('%Y%m%d') endDate_unformatted = datetime.datetime.strptime(startDate, '%Y%m%d') + datetime.timedelta(days=5) endDate = endDate_unformatted.strftime('%Y%m%d') + data = "" fetchUrl = f"https://api.weather.com/v1/geocode/{geocode}/forecast/tides.xml?language=en-US&units=e&startDate={startDate}&endDate={endDate}&apiKey={apiKey}" - res = requests.get(fetchUrl) + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write TideForecast -- status code {r.status}") + return + + data = await r.text() + - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write TidesForecast record -- Status code {res.status_code}") - return - - data = res.text newData = data[53:-16] i2Doc = f'\n \n {newData}\n {tideStation}\n ' - f = open('./.temp/TidesForecast.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): if len(tideStations) < 1: l.debug("Skipping TidesForecast -- No locations.") return @@ -56,21 +57,21 @@ def makeRecord(): header = '' footer = '' - with open('./.temp/TidesForecast.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(tideStations, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/TidesForecast.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/TidesForecast.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/TidesForecast.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/TidesForecast.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip From e1432fd4645d9e0d2607789e63a8a77ed0833d9d Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:11:31 -0700 Subject: [PATCH 27/35] WateringNeeds to asynchronous --- recordGenerators/WateringNeeds.py | 42 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/recordGenerators/WateringNeeds.py b/recordGenerators/WateringNeeds.py index 0436811..86e04e1 100644 --- a/recordGenerators/WateringNeeds.py +++ b/recordGenerators/WateringNeeds.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,47 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/wateringNeeds/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" - - res = requests.get(fetchUrl) - - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write WateringNeeds record -- Status code {res.status_code}") - return + data = "" + + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to WateringNeeds -- status code {r.status}") + return + + data = await r.text() - data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/WateringNeeds.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): l.info("Writing WateringNeeds record.") header = '' footer = '' - with open('./.temp/WateringNeeds.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/WateringNeeds.i2m', 'a') as end: + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as end: end.write(footer) dom = xml.dom.minidom.parse('./.temp/WateringNeeds.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/WateringNeeds.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip From 29420a3a5043e1e550aebb2f5b114f5845243b09 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:21:28 -0700 Subject: [PATCH 28/35] Make new tasks for converted record generators --- RecordTasks.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- main.py | 18 ++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/RecordTasks.py b/RecordTasks.py index d1cf5aa..54b5a11 100644 --- a/RecordTasks.py +++ b/RecordTasks.py @@ -1,5 +1,6 @@ import asyncio -from recordGenerators import Alerts,CurrentObservations,HourlyForecast,DailyForecast +from recordGenerators import Alerts,CurrentObservations,HourlyForecast,DailyForecast, AirQuality, AirportDelays, AchesAndPains, Breathing, HeatingAndCooling, MosquitoActivity, PollenForecast, TideForecast, WateringNeeds +from radar import TWCRadarCollector """ This houses the tasks needed to update the data records concurrently @@ -16,6 +17,8 @@ async def coTask(): await CurrentObservations.makeDataFile() await asyncio.sleep(5 * 60) +# These tasks should be updated every hour + async def hfTask(): while True: await HourlyForecast.makeDataFile() @@ -24,4 +27,49 @@ async def hfTask(): async def dfTask(): while True: await DailyForecast.makeDataFile() + await asyncio.sleep(60 * 60) + +async def aqTask(): + while True: + await AirQuality.writeData() + await asyncio.sleep(60 * 60) + +async def aptTask(): + while True: + await AirportDelays.writeData() + await asyncio.sleep(60 * 60) + +async def apTask(): + while True: + await AchesAndPains.makeRecord() + await asyncio.sleep(60 * 60) + +async def brTask(): + while True: + await Breathing.makeDataFile() + await asyncio.sleep(60 * 60) + +async def hcTask(): + while True: + await HeatingAndCooling.makeRecord() + await asyncio.sleep(60 * 60) + +async def maTask(): + while True: + await MosquitoActivity.makeRecord() + await asyncio.sleep(60 * 60) + +async def pTask(): + while True: + await PollenForecast.makeDataFile() + await asyncio.sleep(60 * 60) + +async def tTask(): + while True: + await TideForecast.makeRecord() + await asyncio.sleep(60 * 60) + +async def wnTask(): + while True: + await WateringNeeds.makeRecord() await asyncio.sleep(60 * 60) \ No newline at end of file diff --git a/main.py b/main.py index 2df865a..9e9cc60 100644 --- a/main.py +++ b/main.py @@ -46,12 +46,30 @@ async def main(): coTask = asyncio.create_task(RecordTasks.coTask()) hfTask = asyncio.create_task(RecordTasks.hfTask()) dfTask = asyncio.create_task(RecordTasks.dfTask()) + aqTask = asyncio.create_task(RecordTasks.aqTask()) + aptTask = asyncio.create_task(RecordTasks.aptTask()) + apTask = asyncio.create_task(RecordTasks.apTask()) + brTask = asyncio.create_task(RecordTasks.brTask()) + hcTask = asyncio.create_task(RecordTasks.hcTask()) + maTask = asyncio.create_task(RecordTasks.maTask()) + pTask = asyncio.create_task(RecordTasks.pTask()) + tTask = asyncio.create_task(RecordTasks.tTask()) + wnTask = asyncio.create_task(RecordTasks.wnTask()) # In theory, these should all run concurrently without problems await alertsTask await coTask await hfTask await dfTask + await aqTask + await aptTask + await apTask + await brTask + await hcTask + await maTask + await pTask + await tTask + await wnTask if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file From 60d672892675312c82239a96f4437863a30de0d8 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:23:53 -0700 Subject: [PATCH 29/35] Fix WateringNeeds not writing footer --- recordGenerators/WateringNeeds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recordGenerators/WateringNeeds.py b/recordGenerators/WateringNeeds.py index 86e04e1..9d58023 100644 --- a/recordGenerators/WateringNeeds.py +++ b/recordGenerators/WateringNeeds.py @@ -54,7 +54,7 @@ async def makeRecord(): await getData(x,y) async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as end: - end.write(footer) + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/WateringNeeds.i2m') xmlPretty = dom.toprettyxml(indent= " ") From ce51e37e7f02e56a747888f3e5a502dd93355e63 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:37:08 -0700 Subject: [PATCH 30/35] Set write mode for getData() to append instead of write --- recordGenerators/HourlyForecast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index 4bca981..e6d2870 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -5,7 +5,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiofiles, aiohttp +import aiofiles, aiohttp, asyncio import sys sys.path.append("./py2lib") @@ -48,7 +48,7 @@ async def getData(tecci, zipCode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' - async with aiofiles.open('./.temp/HourlyForecast.i2m', 'w') as f: + async with aiofiles.open('./.temp/HourlyForecast.i2m', 'a') as f: await f.write(i2Doc) await f.close() @@ -61,6 +61,7 @@ async def makeDataFile(): async with aiofiles.open("./.temp/HourlyForecast.i2m", 'w') as doc: await doc.write(header) + for x, y in zip(tecciId, zipCodes): await getData(x, y) From 175ab43039439a354fcb71ad743d375250e81abe Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:41:51 -0700 Subject: [PATCH 31/35] Make tasks for updating from the radar server --- RecordTasks.py | 22 ++++++++++++++++++++++ main.py | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/RecordTasks.py b/RecordTasks.py index 54b5a11..d3493af 100644 --- a/RecordTasks.py +++ b/RecordTasks.py @@ -1,12 +1,34 @@ import asyncio from recordGenerators import Alerts,CurrentObservations,HourlyForecast,DailyForecast, AirQuality, AirportDelays, AchesAndPains, Breathing, HeatingAndCooling, MosquitoActivity, PollenForecast, TideForecast, WateringNeeds from radar import TWCRadarCollector +from datetime import datetime """ This houses the tasks needed to update the data records concurrently I have no idea if this is a messy way to do things, but it will be worked upon if it is. """ +async def updateMosaicTask(): + mosaicUpdateIntervals = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] + + while True: + # Server takes ~15 - 35 seconds to fully generate a frame, so use 40 seconds to ensure it's fully generated. + if datetime.now().minute in mosaicUpdateIntervals and datetime.now().second == 40: + await TWCRadarCollector.collect("radarmosaic") + + await asyncio.sleep(1) + +async def updateSatradTask(): + satradUpdateIntervals = [0, 10, 20, 30, 40, 50] + + while True: + # Server takes ~15 - 35 seconds to fully generate a frame, so use 40 seconds to ensure it's fully generated. + if datetime.now().minute in satradUpdateIntervals and datetime.now().second == 40: + await TWCRadarCollector.collect("satrad") + + await asyncio.sleep(1) + + async def alertsTask(): while True: await Alerts.makeRecord() diff --git a/main.py b/main.py index 9e9cc60..1c511fe 100644 --- a/main.py +++ b/main.py @@ -42,6 +42,8 @@ async def createTemp(): async def main(): await createTemp() + mosaicTask = asyncio.create_task(RecordTasks.updateMosaicTask()) + satradTask = asyncio.create_task(RecordTasks.updateSatradTask()) alertsTask = asyncio.create_task(RecordTasks.alertsTask()) coTask = asyncio.create_task(RecordTasks.coTask()) hfTask = asyncio.create_task(RecordTasks.hfTask()) @@ -71,5 +73,9 @@ async def main(): await tTask await wnTask + if useRadarServer: + await mosaicTask + await satradTask + if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file From 7727cdc23dbab658810153fb9c68328538507857 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 16:50:54 -0700 Subject: [PATCH 32/35] Replace broken unicode with fixed character --- recordGenerators/AirportDelays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index 7da8b04..3d2678d 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -38,7 +38,7 @@ async def getData(airport): async with s.get(url) as r: data = await r.text() - newData = data[48:-11] + newData = data[48:-11].replace('¿', '-') # Write to i2doc file i2Doc = f'' + '' + newData + f'{airport}' From 2936a64cff11806c9419344f680d580cee838372 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 17:08:26 -0700 Subject: [PATCH 33/35] Prevent blocking when sending files to the i2 --- radar/TWCRadarCollector.py | 3 ++- recordGenerators/AchesAndPains.py | 5 +++-- recordGenerators/AirQuality.py | 5 +++-- recordGenerators/AirportDelays.py | 5 +++-- recordGenerators/Alerts.py | 3 ++- recordGenerators/Breathing.py | 5 +++-- recordGenerators/CurrentObservations.py | 6 +++--- recordGenerators/DailyForecast.py | 5 +++-- recordGenerators/HeatingAndCooling.py | 5 +++-- recordGenerators/HourlyForecast.py | 5 +++-- recordGenerators/MosquitoActivity.py | 5 +++-- recordGenerators/PollenForecast.py | 5 +++-- recordGenerators/TideForecast.py | 5 +++-- recordGenerators/WateringNeeds.py | 5 +++-- 14 files changed, 40 insertions(+), 27 deletions(-) diff --git a/radar/TWCRadarCollector.py b/radar/TWCRadarCollector.py index 30a94b4..705fa49 100644 --- a/radar/TWCRadarCollector.py +++ b/radar/TWCRadarCollector.py @@ -88,6 +88,7 @@ def getTime(timestamp) -> str: return str(time) async def collect(radarType: str): + loop = asyncio.get_running_loop() ts = await getValidTimestamps(radarType) frames = await downloadRadarFrames(radarType, ts) @@ -99,4 +100,4 @@ async def collect(radarType: str): if radarType == "satrad": commands.append( '' ) - bit.sendFile([frames[i]], [commands[i]], 1, 0) + await loop.run_in_executor(bit.sendFile([frames[i]], [commands[i]], 1, 0)) diff --git a/recordGenerators/AchesAndPains.py b/recordGenerators/AchesAndPains.py index e434125..ff686b1 100644 --- a/recordGenerators/AchesAndPains.py +++ b/recordGenerators/AchesAndPains.py @@ -7,7 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -43,6 +43,7 @@ async def getData(coopId, geocode): await f.close() async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing AchesAndPains record.") header = '' @@ -73,7 +74,7 @@ async def makeRecord(): file = "./.temp/AchesAndPains.gz" command = '' - bit.sendFile([file], [command], 1, 0) + await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) remove('./.temp/AchesAndPains.i2m') remove('./.temp/AchesAndPains.gz') \ No newline at end of file diff --git a/recordGenerators/AirQuality.py b/recordGenerators/AirQuality.py index 8334783..94626f9 100644 --- a/recordGenerators/AirQuality.py +++ b/recordGenerators/AirQuality.py @@ -4,7 +4,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -46,6 +46,7 @@ async def getData(epaId, zipcode): await f.close() async def writeData(): + loop = asyncio.get_running_loop() useData = False workingEpaIds = [] @@ -93,7 +94,7 @@ async def writeData(): comand = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/AirQuality.i2m") os.remove("./.temp/AirQuality.gz") diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index 3d2678d..e5b8a0d 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -4,7 +4,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio import sys sys.path.append("./py2lib") @@ -48,6 +48,7 @@ async def getData(airport): await f.close() async def writeData(): + loop = asyncio.get_running_loop() useData = False airportsWithDelays = [] @@ -93,7 +94,7 @@ async def writeData(): comand = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/AirportDelays.i2m") os.remove("./.temp/AirportDelays.gz") diff --git a/recordGenerators/Alerts.py b/recordGenerators/Alerts.py index 2a00f63..4386a84 100644 --- a/recordGenerators/Alerts.py +++ b/recordGenerators/Alerts.py @@ -9,7 +9,7 @@ import xml.dom.minidom import shutil import gzip import logging,coloredlogs -import aiofiles, aiohttp +import aiohttp, aiofiles, asyncio import sys @@ -324,6 +324,7 @@ async def getAlerts(location): async def makeRecord(): + loop = asyncio.get_running_loop() global k # The BERecord XML doesn't need to be written if there's no alerts. diff --git a/recordGenerators/Breathing.py b/recordGenerators/Breathing.py index 37e79b9..d5b6ee8 100644 --- a/recordGenerators/Breathing.py +++ b/recordGenerators/Breathing.py @@ -6,7 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio sys.path.append("./py2lib") sys.path.append("./Util") @@ -52,6 +52,7 @@ async def getData(coopId, geocode): async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a Breathing forecast record.") header = '' footer = '' @@ -85,7 +86,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/Breathing.i2m") os.remove("./.temp/Breathing.gz") \ No newline at end of file diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index f0ca993..ac30fc1 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -6,8 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiofiles -import aiohttp +import aiohttp, aiofiles, asyncio import sys sys.path.append("./py2lib") @@ -55,6 +54,7 @@ async def getData(tecci, zipCode): async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a CurrentObservations record.") header = '' footer = '' @@ -92,7 +92,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0))) os.remove("./.temp/CurrentObservations.i2m") os.remove("./.temp/CurrentObservations.gz") diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index 9950f69..0d9ea99 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -6,7 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiofiles, aiohttp +import aiohttp, aiofiles, asyncio sys.path.append("./py2lib") sys.path.append("./Util") @@ -53,6 +53,7 @@ async def getData(tecci, zipCode): async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a DailyForecast record.") header = '' footer = '' @@ -86,7 +87,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/DailyForecast.i2m") os.remove("./.temp/DailyForecast.gz") \ No newline at end of file diff --git a/recordGenerators/HeatingAndCooling.py b/recordGenerators/HeatingAndCooling.py index 787739e..3b07a3b 100644 --- a/recordGenerators/HeatingAndCooling.py +++ b/recordGenerators/HeatingAndCooling.py @@ -7,7 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -43,6 +43,7 @@ async def getData(coopId, geocode): await f.close() async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing HeatingAndCooling record.") header = '' @@ -73,7 +74,7 @@ async def makeRecord(): file = "./.temp/HeatingAndCooling.gz" command = '' - bit.sendFile([file], [command], 1, 0) + await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) remove('./.temp/HeatingAndCooling.i2m') remove('./.temp/HeatingAndCooling.gz') \ No newline at end of file diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index e6d2870..eaf76d3 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -5,7 +5,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs -import aiofiles, aiohttp, asyncio +import aiohttp, aiofiles, asyncio, asyncio import sys sys.path.append("./py2lib") @@ -54,6 +54,7 @@ async def getData(tecci, zipCode): async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing an HourlyForecast record.") header = '' footer = '' @@ -88,7 +89,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/HourlyForecast.i2m") os.remove("./.temp/HourlyForecast.gz") \ No newline at end of file diff --git a/recordGenerators/MosquitoActivity.py b/recordGenerators/MosquitoActivity.py index cb22f47..d3fa07f 100644 --- a/recordGenerators/MosquitoActivity.py +++ b/recordGenerators/MosquitoActivity.py @@ -7,7 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -43,6 +43,7 @@ async def getData(coopId, geocode): await f.close() async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing MosquitoActivity record.") header = '' @@ -73,7 +74,7 @@ async def makeRecord(): file = "./.temp/MosquitoActivity.gz" command = '' - bit.sendFile([file], [command], 1, 0) + await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) remove('./.temp/MosquitoActivity.i2m') remove('./.temp/MosquitoActivity.gz') \ No newline at end of file diff --git a/recordGenerators/PollenForecast.py b/recordGenerators/PollenForecast.py index 7c3f475..4fc90bc 100644 --- a/recordGenerators/PollenForecast.py +++ b/recordGenerators/PollenForecast.py @@ -6,7 +6,7 @@ import os import shutil import xml.dom.minidom import logging, coloredlogs -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio sys.path.append("./py2lib") sys.path.append("./Util") @@ -52,6 +52,7 @@ async def getData(pollenId, geocode): async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a PollenForecast record.") header = '' footer = '' @@ -85,7 +86,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/PollenForecast.i2m") os.remove("./.temp/PollenForecast.gz") \ No newline at end of file diff --git a/recordGenerators/TideForecast.py b/recordGenerators/TideForecast.py index bb69863..d358190 100644 --- a/recordGenerators/TideForecast.py +++ b/recordGenerators/TideForecast.py @@ -7,7 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -48,6 +48,7 @@ async def getData(tideStation, geocode): await f.close() async def makeRecord(): + loop = asyncio.get_running_loop() if len(tideStations) < 1: l.debug("Skipping TidesForecast -- No locations.") return @@ -82,7 +83,7 @@ async def makeRecord(): file = "./.temp/TidesForecast.gz" command = '' - bit.sendFile([file], [command], 1, 0) + await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) remove('./.temp/TidesForecast.i2m') remove('./.temp/TidesForecast.gz') \ No newline at end of file diff --git a/recordGenerators/WateringNeeds.py b/recordGenerators/WateringNeeds.py index 9d58023..e464f0e 100644 --- a/recordGenerators/WateringNeeds.py +++ b/recordGenerators/WateringNeeds.py @@ -7,7 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom -import aiohttp, aiofiles +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -42,6 +42,7 @@ async def getData(coopId, geocode): await f.close() async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing WateringNeeds record.") header = '' @@ -72,7 +73,7 @@ async def makeRecord(): file = "./.temp/WateringNeeds.gz" command = '' - bit.sendFile([file], [command], 1, 0) + await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) remove('./.temp/WateringNeeds.i2m') remove('./.temp/WateringNeeds.gz') \ No newline at end of file From 08da432ea20b5538f882a135123f448a02248e1d Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 17:11:55 -0700 Subject: [PATCH 34/35] Fix syntax of previous commit --- recordGenerators/AchesAndPains.py | 2 +- recordGenerators/AirQuality.py | 2 +- recordGenerators/AirportDelays.py | 2 +- recordGenerators/Breathing.py | 2 +- recordGenerators/CurrentObservations.py | 2 +- recordGenerators/DailyForecast.py | 2 +- recordGenerators/HeatingAndCooling.py | 2 +- recordGenerators/HourlyForecast.py | 2 +- recordGenerators/MosquitoActivity.py | 2 +- recordGenerators/PollenForecast.py | 2 +- recordGenerators/TideForecast.py | 2 +- recordGenerators/WateringNeeds.py | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/recordGenerators/AchesAndPains.py b/recordGenerators/AchesAndPains.py index ff686b1..6c07aa3 100644 --- a/recordGenerators/AchesAndPains.py +++ b/recordGenerators/AchesAndPains.py @@ -74,7 +74,7 @@ async def makeRecord(): file = "./.temp/AchesAndPains.gz" command = '' - await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) + await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) remove('./.temp/AchesAndPains.i2m') remove('./.temp/AchesAndPains.gz') \ No newline at end of file diff --git a/recordGenerators/AirQuality.py b/recordGenerators/AirQuality.py index 94626f9..aeb556b 100644 --- a/recordGenerators/AirQuality.py +++ b/recordGenerators/AirQuality.py @@ -94,7 +94,7 @@ async def writeData(): comand = commands.append('') numFiles = len(files) - await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) + await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/AirQuality.i2m") os.remove("./.temp/AirQuality.gz") diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index e5b8a0d..e9337ed 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -94,7 +94,7 @@ async def writeData(): comand = commands.append('') numFiles = len(files) - await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) + await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/AirportDelays.i2m") os.remove("./.temp/AirportDelays.gz") diff --git a/recordGenerators/Breathing.py b/recordGenerators/Breathing.py index d5b6ee8..097ed79 100644 --- a/recordGenerators/Breathing.py +++ b/recordGenerators/Breathing.py @@ -86,7 +86,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) + await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/Breathing.i2m") os.remove("./.temp/Breathing.gz") \ No newline at end of file diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index ac30fc1..724fdb8 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -92,7 +92,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0))) + await loop.run_in_executor(await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0))) os.remove("./.temp/CurrentObservations.i2m") os.remove("./.temp/CurrentObservations.gz") diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index 0d9ea99..e0087d7 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -87,7 +87,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) + await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/DailyForecast.i2m") os.remove("./.temp/DailyForecast.gz") \ No newline at end of file diff --git a/recordGenerators/HeatingAndCooling.py b/recordGenerators/HeatingAndCooling.py index 3b07a3b..f2bdd33 100644 --- a/recordGenerators/HeatingAndCooling.py +++ b/recordGenerators/HeatingAndCooling.py @@ -74,7 +74,7 @@ async def makeRecord(): file = "./.temp/HeatingAndCooling.gz" command = '' - await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) + await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) remove('./.temp/HeatingAndCooling.i2m') remove('./.temp/HeatingAndCooling.gz') \ No newline at end of file diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index eaf76d3..c730154 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -89,7 +89,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) + await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/HourlyForecast.i2m") os.remove("./.temp/HourlyForecast.gz") \ No newline at end of file diff --git a/recordGenerators/MosquitoActivity.py b/recordGenerators/MosquitoActivity.py index d3fa07f..7463e81 100644 --- a/recordGenerators/MosquitoActivity.py +++ b/recordGenerators/MosquitoActivity.py @@ -74,7 +74,7 @@ async def makeRecord(): file = "./.temp/MosquitoActivity.gz" command = '' - await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) + await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) remove('./.temp/MosquitoActivity.i2m') remove('./.temp/MosquitoActivity.gz') \ No newline at end of file diff --git a/recordGenerators/PollenForecast.py b/recordGenerators/PollenForecast.py index 4fc90bc..c8a91fc 100644 --- a/recordGenerators/PollenForecast.py +++ b/recordGenerators/PollenForecast.py @@ -86,7 +86,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) + await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/PollenForecast.i2m") os.remove("./.temp/PollenForecast.gz") \ No newline at end of file diff --git a/recordGenerators/TideForecast.py b/recordGenerators/TideForecast.py index d358190..e36230f 100644 --- a/recordGenerators/TideForecast.py +++ b/recordGenerators/TideForecast.py @@ -83,7 +83,7 @@ async def makeRecord(): file = "./.temp/TidesForecast.gz" command = '' - await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) + await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) remove('./.temp/TidesForecast.i2m') remove('./.temp/TidesForecast.gz') \ No newline at end of file diff --git a/recordGenerators/WateringNeeds.py b/recordGenerators/WateringNeeds.py index e464f0e..4f5c586 100644 --- a/recordGenerators/WateringNeeds.py +++ b/recordGenerators/WateringNeeds.py @@ -73,7 +73,7 @@ async def makeRecord(): file = "./.temp/WateringNeeds.gz" command = '' - await loop.run_in_executor(bit.sendFile([file], [command], 1, 0)) + await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) remove('./.temp/WateringNeeds.i2m') remove('./.temp/WateringNeeds.gz') \ No newline at end of file From 4645bc5a53c64cf6ca9b8f678fb27a740b9ba9b9 Mon Sep 17 00:00:00 2001 From: April Date: Sun, 13 Nov 2022 17:19:09 -0700 Subject: [PATCH 35/35] Undo previous commits --- radar/TWCRadarCollector.py | 2 +- recordGenerators/AchesAndPains.py | 2 +- recordGenerators/AirQuality.py | 2 +- recordGenerators/AirportDelays.py | 2 +- recordGenerators/Breathing.py | 2 +- recordGenerators/CurrentObservations.py | 2 +- recordGenerators/DailyForecast.py | 2 +- recordGenerators/HeatingAndCooling.py | 2 +- recordGenerators/HourlyForecast.py | 2 +- recordGenerators/MosquitoActivity.py | 2 +- recordGenerators/PollenForecast.py | 2 +- recordGenerators/TideForecast.py | 2 +- recordGenerators/WateringNeeds.py | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/radar/TWCRadarCollector.py b/radar/TWCRadarCollector.py index 705fa49..65cbef4 100644 --- a/radar/TWCRadarCollector.py +++ b/radar/TWCRadarCollector.py @@ -100,4 +100,4 @@ async def collect(radarType: str): if radarType == "satrad": commands.append( '' ) - await loop.run_in_executor(bit.sendFile([frames[i]], [commands[i]], 1, 0)) + bit.sendFile([frames[i]], [commands[i]], 1, 0) diff --git a/recordGenerators/AchesAndPains.py b/recordGenerators/AchesAndPains.py index 6c07aa3..172c3e5 100644 --- a/recordGenerators/AchesAndPains.py +++ b/recordGenerators/AchesAndPains.py @@ -74,7 +74,7 @@ async def makeRecord(): file = "./.temp/AchesAndPains.gz" command = '' - await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) + bit.sendFile([file], [command], 1, 0) remove('./.temp/AchesAndPains.i2m') remove('./.temp/AchesAndPains.gz') \ No newline at end of file diff --git a/recordGenerators/AirQuality.py b/recordGenerators/AirQuality.py index aeb556b..a1f5019 100644 --- a/recordGenerators/AirQuality.py +++ b/recordGenerators/AirQuality.py @@ -94,7 +94,7 @@ async def writeData(): comand = commands.append('') numFiles = len(files) - await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) + bit.sendFile(files, commands, numFiles, 0) os.remove("./.temp/AirQuality.i2m") os.remove("./.temp/AirQuality.gz") diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index e9337ed..b5b941a 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -94,7 +94,7 @@ async def writeData(): comand = commands.append('') numFiles = len(files) - await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) + bit.sendFile(files, commands, numFiles, 0) os.remove("./.temp/AirportDelays.i2m") os.remove("./.temp/AirportDelays.gz") diff --git a/recordGenerators/Breathing.py b/recordGenerators/Breathing.py index 097ed79..991c85b 100644 --- a/recordGenerators/Breathing.py +++ b/recordGenerators/Breathing.py @@ -86,7 +86,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) + bit.sendFile(files, commands, numFiles, 0) os.remove("./.temp/Breathing.i2m") os.remove("./.temp/Breathing.gz") \ No newline at end of file diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index 724fdb8..b41da7a 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -92,7 +92,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0))) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/CurrentObservations.i2m") os.remove("./.temp/CurrentObservations.gz") diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index e0087d7..5becbfe 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -87,7 +87,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) + bit.sendFile(files, commands, numFiles, 0) os.remove("./.temp/DailyForecast.i2m") os.remove("./.temp/DailyForecast.gz") \ No newline at end of file diff --git a/recordGenerators/HeatingAndCooling.py b/recordGenerators/HeatingAndCooling.py index f2bdd33..e2790bb 100644 --- a/recordGenerators/HeatingAndCooling.py +++ b/recordGenerators/HeatingAndCooling.py @@ -74,7 +74,7 @@ async def makeRecord(): file = "./.temp/HeatingAndCooling.gz" command = '' - await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) + bit.sendFile([file], [command], 1, 0) remove('./.temp/HeatingAndCooling.i2m') remove('./.temp/HeatingAndCooling.gz') \ No newline at end of file diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index c730154..d1ea73e 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -89,7 +89,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) + bit.sendFile(files, commands, numFiles, 0) os.remove("./.temp/HourlyForecast.i2m") os.remove("./.temp/HourlyForecast.gz") \ No newline at end of file diff --git a/recordGenerators/MosquitoActivity.py b/recordGenerators/MosquitoActivity.py index 7463e81..dba9acd 100644 --- a/recordGenerators/MosquitoActivity.py +++ b/recordGenerators/MosquitoActivity.py @@ -74,7 +74,7 @@ async def makeRecord(): file = "./.temp/MosquitoActivity.gz" command = '' - await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) + bit.sendFile([file], [command], 1, 0) remove('./.temp/MosquitoActivity.i2m') remove('./.temp/MosquitoActivity.gz') \ No newline at end of file diff --git a/recordGenerators/PollenForecast.py b/recordGenerators/PollenForecast.py index c8a91fc..960b2e1 100644 --- a/recordGenerators/PollenForecast.py +++ b/recordGenerators/PollenForecast.py @@ -86,7 +86,7 @@ async def makeDataFile(): command = commands.append('') numFiles = len(files) - await loop.run_in_executor(None, bit.sendFile(files, commands, numFiles, 0)) + bit.sendFile(files, commands, numFiles, 0) os.remove("./.temp/PollenForecast.i2m") os.remove("./.temp/PollenForecast.gz") \ No newline at end of file diff --git a/recordGenerators/TideForecast.py b/recordGenerators/TideForecast.py index e36230f..3dce2a7 100644 --- a/recordGenerators/TideForecast.py +++ b/recordGenerators/TideForecast.py @@ -83,7 +83,7 @@ async def makeRecord(): file = "./.temp/TidesForecast.gz" command = '' - await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) + bit.sendFile([file], [command], 1, 0) remove('./.temp/TidesForecast.i2m') remove('./.temp/TidesForecast.gz') \ No newline at end of file diff --git a/recordGenerators/WateringNeeds.py b/recordGenerators/WateringNeeds.py index 4f5c586..7a3c494 100644 --- a/recordGenerators/WateringNeeds.py +++ b/recordGenerators/WateringNeeds.py @@ -73,7 +73,7 @@ async def makeRecord(): file = "./.temp/WateringNeeds.gz" command = '' - await loop.run_in_executor(None, bit.sendFile([file], [command], 1, 0)) + bit.sendFile([file], [command], 1, 0) remove('./.temp/WateringNeeds.i2m') remove('./.temp/WateringNeeds.gz') \ No newline at end of file