diff --git a/RecordTasks.py b/RecordTasks.py new file mode 100644 index 0000000..d3493af --- /dev/null +++ b/RecordTasks.py @@ -0,0 +1,97 @@ +import asyncio +from recordGenerators import Alerts,CurrentObservations,HourlyForecast,DailyForecast, AirQuality, AirportDelays, AchesAndPains, Breathing, HeatingAndCooling, MosquitoActivity, PollenForecast, TideForecast, WateringNeeds +from radar import TWCRadarCollector +from datetime import datetime + + +""" This houses the tasks needed to update the data records concurrently + I have no idea if this is a messy way to do things, but it will be worked upon if it is. +""" + +async def updateMosaicTask(): + mosaicUpdateIntervals = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] + + while True: + # Server takes ~15 - 35 seconds to fully generate a frame, so use 40 seconds to ensure it's fully generated. + if datetime.now().minute in mosaicUpdateIntervals and datetime.now().second == 40: + await TWCRadarCollector.collect("radarmosaic") + + await asyncio.sleep(1) + +async def updateSatradTask(): + satradUpdateIntervals = [0, 10, 20, 30, 40, 50] + + while True: + # Server takes ~15 - 35 seconds to fully generate a frame, so use 40 seconds to ensure it's fully generated. + if datetime.now().minute in satradUpdateIntervals and datetime.now().second == 40: + await TWCRadarCollector.collect("satrad") + + await asyncio.sleep(1) + + +async def alertsTask(): + while True: + await Alerts.makeRecord() + await asyncio.sleep(60) + +async def coTask(): + while True: + await CurrentObservations.makeDataFile() + await asyncio.sleep(5 * 60) + +# These tasks should be updated every hour + +async def hfTask(): + while True: + await HourlyForecast.makeDataFile() + await asyncio.sleep(60 * 60) + +async def dfTask(): + while True: + await DailyForecast.makeDataFile() + await asyncio.sleep(60 * 60) + +async def aqTask(): + while True: + await AirQuality.writeData() + await asyncio.sleep(60 * 60) + +async def aptTask(): + while True: + await AirportDelays.writeData() + await asyncio.sleep(60 * 60) + +async def apTask(): + while True: + await AchesAndPains.makeRecord() + await asyncio.sleep(60 * 60) + +async def brTask(): + while True: + await Breathing.makeDataFile() + await asyncio.sleep(60 * 60) + +async def hcTask(): + while True: + await HeatingAndCooling.makeRecord() + await asyncio.sleep(60 * 60) + +async def maTask(): + while True: + await MosquitoActivity.makeRecord() + await asyncio.sleep(60 * 60) + +async def pTask(): + while True: + await PollenForecast.makeDataFile() + await asyncio.sleep(60 * 60) + +async def tTask(): + while True: + await TideForecast.makeRecord() + await asyncio.sleep(60 * 60) + +async def wnTask(): + while True: + await WateringNeeds.makeRecord() + await asyncio.sleep(60 * 60) \ No newline at end of file diff --git a/Util/MachineProductCfg.py b/Util/MachineProductCfg.py index b6d4970..6549eb2 100644 --- a/Util/MachineProductCfg.py +++ b/Util/MachineProductCfg.py @@ -50,9 +50,90 @@ def getTideStations(): def getAirportCodes(): """ Returns all of the airport identifiers present in the MachineProductCfg """ - airports = [] + airports = [ + 'ATL', + 'LAX', + 'ORD', + 'DFW', + 'JFK', + 'DEN', + 'SFO', + 'CLT', + 'LAS', + 'PHX', + 'IAH', + 'MIA', + 'SEA', + 'EWR', + 'MCO', + 'MSP', + 'DTW', + 'BOS', + 'PHL', + 'LGA', + 'FLL', + 'BWI', + 'IAD', + 'MDW', + 'SLC', + 'DCA', + 'HNL', + 'SAN', + 'TPA', + 'PDX', + 'STL', + 'HOU', + 'BNA', + 'AUS', + 'OAK', + 'MSY', + 'RDU', + 'SJC', + 'SNA', + 'DAL', + 'SMF', + 'SAT', + 'RSW', + 'PIT', + 'CLE', + 'IND', + 'MKE', + 'CMH', + 'OGG', + 'PBI', + 'BDL', + 'CVG', + 'JAX', + 'ANC', + 'BUF', + 'ABQ', + 'ONT', + 'OMA', + 'BUR', + 'OKC', + 'MEM', + 'PVD', + 'RIC', + 'SDF', + 'RNO', + 'TUS', + 'CHS', + 'ORF', + 'PWM', + 'GRR', + 'BHM', + 'LIT', + 'DSM', + 'FAR', + 'FSD', + 'ICT', + 'LBB', + 'BIL', + 'BOI', + 'GEG' + ] for i in data['Config']['ConfigDef']['ConfigItems']['ConfigItem']: - if "Airport" in i['@key'] and i['@value'] != "": + if "Airport" in i['@key'] and i['@value'] != "" and not i['@value'] in airports: # Split the string up airports.append(i['@value'].split("_")[2]) diff --git a/main.py b/main.py index 3520f11..1c511fe 100644 --- a/main.py +++ b/main.py @@ -1,93 +1,81 @@ -import asyncio +import asyncio, aiofiles from asyncio.log import logger from asyncore import loop import logging,coloredlogs -from recordGenerators import Alerts,CurrentObservations,DailyForecast,HourlyForecast,AirportDelays,AirQuality,HeatingAndCooling,PollenForecast,Breathing, AchesAndPains, MosquitoActivity, WateringNeeds, TideForecast from radar import TWCRadarCollector import os from datetime import datetime +import RecordTasks l = logging.getLogger(__name__) coloredlogs.install(logger=l) useRadarServer = True -# Create dirs and files -if not os.path.exists('.temp/'): - os.makedirs('.temp/') - -if not os.path.exists('.temp/tiles/'): - os.makedirs('.temp/tiles/') - -if not os.path.exists('.temp/tiles/output/'): - os.makedirs('.temp/tiles/output/') - -if not os.path.exists('.temp/msgId.txt'): - print("Creating initial msgId file") - with open('.temp/msgId.txt', "w") as f: - f.write("410080515") - - -""" -CurrentConditions: Every 5 minutes -Daily Forecasts, Hourlies, etc: 60 minutes -Alerts: 5 minutes -""" l.info("Starting i2RecordCollector") l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose") -async def grabAlertsLoop(): - while True: - Alerts.makeRecord() - await asyncio.sleep(60) - -async def FiveMinUpdaters(): - while True: - CurrentObservations.makeDataFile() - l.debug("Sleeping for 5 minutes...") - await asyncio.sleep(5 * 60) - -async def HourUpdaters(): - while True: - DailyForecast.makeDataFile() - HourlyForecast.makeDataFile() - AirQuality.writeData() - PollenForecast.makeDataFile() - AirportDelays.writeData() - Breathing.makeDataFile() - HeatingAndCooling.makeRecord() - WateringNeeds.makeRecord() - MosquitoActivity.makeRecord() - AchesAndPains.makeRecord() - TideForecast.makeRecord() - l.debug("Sleeping for an hour...") - await asyncio.sleep(60 * 60) - -async def radarCollector(): - mosaicUpdateIntervals = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] - satradUpdateIntervals = [0, 10, 20, 30, 40, 50] - - while True: - # Server takes ~15 - 35 seconds on average to fully generate a frame, use 40 seconds - # to make sure the radar frame is fully good to go - if datetime.now().minute in mosaicUpdateIntervals and datetime.now().second == 40: - await TWCRadarCollector.collect("radarmosaic") - - if datetime.now().minute in satradUpdateIntervals and datetime.now().second == 45: - await TWCRadarCollector.collect("satrad") +async def createTemp(): + """ Used on a first time run, creates necessary files & directories for the message encoder to work properly. """ + if not (os.path.exists('./.temp/')): + l.info("Creating necessary directories & files..") + os.mkdir('./.temp') - await asyncio.sleep(1) + # Used for the record generator + os.mkdir('./.temp/tiles/') + os.mkdir('./.temp/tiles/output/') -loop = asyncio.get_event_loop() -alertTask = loop.create_task(grabAlertsLoop()) -CCtask = loop.create_task(FiveMinUpdaters()) -ForecastsTask = loop.create_task(HourUpdaters()) + # Used for radar server downloads + os.mkdir('./.temp/output') + os.mkdir('./.temp/output/radarmosaic') + os.mkdir('./.temp/output/satrad') -if useRadarServer: radarTask = loop.create_task(radarCollector()) + # Create msgId file for bit.py + async with aiofiles.open('./.temp/msgId.txt', 'w') as msgId: + await msgId.write('410080515') + await msgId.close() + else: + l.debug(".temp file exists") + return -try: - loop.run_until_complete(alertTask) - loop.run_until_complete(CCtask) - loop.run_until_complete(ForecastsTask) - if useRadarServer: loop.run_until_complete(radarTask) -except asyncio.CancelledError: pass \ No newline at end of file + +async def main(): + await createTemp() + + mosaicTask = asyncio.create_task(RecordTasks.updateMosaicTask()) + satradTask = asyncio.create_task(RecordTasks.updateSatradTask()) + alertsTask = asyncio.create_task(RecordTasks.alertsTask()) + coTask = asyncio.create_task(RecordTasks.coTask()) + hfTask = asyncio.create_task(RecordTasks.hfTask()) + dfTask = asyncio.create_task(RecordTasks.dfTask()) + aqTask = asyncio.create_task(RecordTasks.aqTask()) + aptTask = asyncio.create_task(RecordTasks.aptTask()) + apTask = asyncio.create_task(RecordTasks.apTask()) + brTask = asyncio.create_task(RecordTasks.brTask()) + hcTask = asyncio.create_task(RecordTasks.hcTask()) + maTask = asyncio.create_task(RecordTasks.maTask()) + pTask = asyncio.create_task(RecordTasks.pTask()) + tTask = asyncio.create_task(RecordTasks.tTask()) + wnTask = asyncio.create_task(RecordTasks.wnTask()) + + # In theory, these should all run concurrently without problems + await alertsTask + await coTask + await hfTask + await dfTask + await aqTask + await aptTask + await apTask + await brTask + await hcTask + await maTask + await pTask + await tTask + await wnTask + + if useRadarServer: + await mosaicTask + await satradTask + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/py2Lib/starbundle.py b/py2Lib/starbundle.py index ae00c95..306cfd8 100644 --- a/py2Lib/starbundle.py +++ b/py2Lib/starbundle.py @@ -42,4 +42,4 @@ def makeStarBundle(Directory, Type, flag, Version, date, sendAfter): #Directory which contains Files to be bundled Type Flags Version Date SendImmediately(Does not apply to this script) -makeStarBundle('./.temp/i2State/SD/Managed/Events', 'Managed', 'Domestic_SD_Universe', '637898877227230030', '09/28/2022', 0) \ No newline at end of file +# makeStarBundle('./.temp/i2State/SD/Managed/Events', 'Managed', 'Domestic_SD_Universe', '637898877227230030', '09/28/2022', 0) \ No newline at end of file diff --git a/radar/RadarProcessor.py b/radar/RadarProcessor.py index 9e1967b..65d62bf 100644 --- a/radar/RadarProcessor.py +++ b/radar/RadarProcessor.py @@ -55,8 +55,8 @@ def WorldCoordinateToTile(coord: Point) -> Point: scale = 1 << 6 return Point( - x = math.floor(coord.x * scale / 256), - y = math.floor(coord.y * scale / 256) + x = math.floor(coord.x * scale / 255), + y = math.floor(coord.y * scale / 255) ) def WorldCoordinateToPixel(coord: Point) -> Point: @@ -68,10 +68,10 @@ def WorldCoordinateToPixel(coord: Point) -> Point: ) def LatLongProject(lat, long) -> Point: - siny = math.sin(lat * math.pi / 185) + siny = math.sin(lat * math.pi / 180) siny = min(max(siny, -0.9999), 0.9999) return Point( x = 256 * (0.5 + long / 360), y = 256 * (0.5 - math.log((1 + siny) / (1 - siny)) / (4 * math.pi)) - ) + ) \ No newline at end of file diff --git a/radar/TWCRadarCollector.py b/radar/TWCRadarCollector.py index 30a94b4..65cbef4 100644 --- a/radar/TWCRadarCollector.py +++ b/radar/TWCRadarCollector.py @@ -88,6 +88,7 @@ def getTime(timestamp) -> str: return str(time) async def collect(radarType: str): + loop = asyncio.get_running_loop() ts = await getValidTimestamps(radarType) frames = await downloadRadarFrames(radarType, ts) diff --git a/recordGenerators/AchesAndPains.py b/recordGenerators/AchesAndPains.py index 6631ca1..172c3e5 100644 --- a/recordGenerators/AchesAndPains.py +++ b/recordGenerators/AchesAndPains.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,49 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/achePain/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" + data = "" + + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write AchesAndPains record -- status code {r.status}") + return + + data = await r.text() - res = requests.get(fetchUrl) - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write AchesAndPains record -- Status code {res.status_code}") - return - - data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/AchesAndPains.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing AchesAndPains record.") header = '' footer = '' - with open('./.temp/AchesAndPains.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/AchesAndPains.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/AchesAndPains.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/AchesAndPains.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/AchesAndPains.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip diff --git a/recordGenerators/AirQuality.py b/recordGenerators/AirQuality.py index cb09528..a1f5019 100644 --- a/recordGenerators/AirQuality.py +++ b/recordGenerators/AirQuality.py @@ -4,6 +4,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -27,22 +28,25 @@ for i in MPC.getPrimaryLocations(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(epaId, zipcode): +async def getData(epaId, zipcode): url = f"https://api.weather.com/v1/location/{zipcode}:4:US/airquality.xml?language=en-US&apiKey={apiKey}" + data = "" - res = requests.get(url=url) - - data = res.text + async with aiohttp.ClientSession() as s: + async with s.get(url) as r: + data = await r.text() + newData = data[57:-11] # Write to i2doc file i2Doc = f'' + '' + newData + f'{epaId}' - f = open("./.temp/AirQuality.i2m", 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/AirQuality.i2m", 'a') as f: + await f.write(i2Doc) + await f.close() -def writeData(): +async def writeData(): + loop = asyncio.get_running_loop() useData = False workingEpaIds = [] @@ -62,21 +66,21 @@ def writeData(): header = '' footer = "" - with open("./.temp/AirQuality.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/AirQuality.i2m", 'w') as doc: + await doc.write(header) for (x, y) in zip(workingEpaIds, zipCodes): - getData(x, y) + await getData(x, y) - with open("./.temp/AirQuality.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/AirQuality.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/AirQuality.i2m") xmlPretty = dom.toprettyxml(indent = " ") - with open("./.temp/AirQuality.i2m", 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open("./.temp/AirQuality.i2m", 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() files = [] commands = [] diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index 4edae1c..b5b941a 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -4,6 +4,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles, asyncio import sys sys.path.append("./py2lib") @@ -29,56 +30,57 @@ l.debug(airports) apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(airport): +async def getData(airport): url = f"https://api.weather.com/v1/airportcode/{airport}/airport/delays.xml?language=en-US&apiKey={apiKey}" + data = "" - res = requests.get(url=url) + async with aiohttp.ClientSession() as s: + async with s.get(url) as r: + data = await r.text() - data = res.text - newData = data[48:-11] + newData = data[48:-11].replace('¿', '-') # Write to i2doc file i2Doc = f'' + '' + newData + f'{airport}' - f = open("./.temp/AirportDelays.i2m", 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/AirportDelays.i2m", 'a') as f: + await f.write(i2Doc) + await f.close() -def writeData(): +async def writeData(): + loop = asyncio.get_running_loop() useData = False airportsWithDelays = [] for x in airports: - # Do a quick check to see if the airport in question has a delay or not - res = requests.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") - - if (res.status_code != 200): - l.debug(f"[AIRPORT DELAYS] No delays for {x} found, skipping..") - else: - airportsWithDelays.append(x) - l.debug(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..") - useData = True + async with aiohttp.ClientSession() as s: + async with s.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") as r: + if r.status != 200: + l.debug(f"No delay for {x} found, skipping..") + else: + airportsWithDelays.append(x) + useData = True if (useData): l.info("Writing an AirportDelays record.") header = '' footer = "" - with open("./.temp/AirportDelays.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/AirportDelays.i2m", 'w') as doc: + await doc.write(header) for x in airportsWithDelays: - getData(x) + await getData(x) - with open("./.temp/AirportDelays.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/AirportDelays.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/AirportDelays.i2m") prettyXml = dom.toprettyxml(indent=" ") - with open("./.temp/AirportDelays.i2m", 'w') as g: - g.write(prettyXml) - g.close() + async with aiofiles.open("./.temp/AirportDelays.i2m", 'w') as g: + await g.write(prettyXml) + await g.close() files = [] commands = [] diff --git a/recordGenerators/Alerts.py b/recordGenerators/Alerts.py index 25c88f8..4386a84 100644 --- a/recordGenerators/Alerts.py +++ b/recordGenerators/Alerts.py @@ -9,6 +9,7 @@ import xml.dom.minidom import shutil import gzip import logging,coloredlogs +import aiohttp, aiofiles, asyncio import sys @@ -26,321 +27,326 @@ headlineApiKey = '21d8a80b3d6b444998a80b3d6b1449d3' detailsApiKey = '21d8a80b3d6b444998a80b3d6b1449d3' k = 0 -def getAlerts(location): +async def getAlerts(location): global k fetchUrl = 'https://api.weather.com/v3/alerts/headlines?areaId=' + location + ':US&format=json&language=en-US&apiKey=' + headlineApiKey - response = requests.get(fetchUrl) + # response = requests.get(fetchUrl) - theCode = response.status_code - - #Our global variables - - - #Set the actions based on response code - if theCode == 204: - l.info('No alerts for area ' + location + '.\n') - return - elif theCode == 403: - l.critical("Uh oh! Your API key may not be authorized for alerts. Tsk Tsk. Maybe you shouldn't pirate IBM data :)\n") - return - elif theCode == 401: - l.critical("Uh oh! This request requires authentication. Maybe you shouldn't try to access resources for IBM employee's only :)\n") - return - elif theCode == 404: - l.error("Uh oh! The requested resource cannot be found. This means either the URL is wrong or IBM is having technical difficulties :(\n Or.... They deleted the API :O\n") - return - elif theCode == 405: - l.error("Uh oh! Got a 405! This means that somehow.... someway..... this script made an invalid request. So sad..... So terrible..... :(\n") - return - elif theCode == 406: - l.critical("Uh oh! Got a 406! This means that IBM doesn't like us. :(\n") - return - elif theCode == 408: - l.error("Uh oh! We were too slow in providing IBM our alert request. Although I prefer to say we were Slowly Capable! :)\n") - return - elif theCode == 500: - l.error("Uh oh! Seems IBM's on call IT Tech spilled coffee on the server! Looks like no alerts for a while. Please check back later :)\n") - return - elif theCode == 502 or theCode == 503 or theCode == 504: - l.error("Uh oh! This is why you don't have interns messing with the server configuration. Please stand by while IBM's on call IT Tech resolves the issue :)\n") - return - elif theCode == 200: - pass + # theCode = response.status_code - #Alright lets map our headline variables. - l.debug('Found Alert for ' + location + '\n') - dataH = response.json() - alertsRoot = dataH['alerts'] + theCode = 0 - for x in alertsRoot: - detailKey = x['detailKey'] - #Lets get map our detail variables. - detailsUrl = 'https://api.weather.com/v3/alerts/detail?alertId=' + detailKey + '&format=json&language=en-US&apiKey=' + detailsApiKey - detailsResponse = requests.get(detailsUrl) - dataD = detailsResponse.json() - detailsRoot = dataD['alertDetail'] - theDetailsText = detailsRoot['texts'] - detailsText = theDetailsText[0] - descriptionRaw = detailsText['description'] - language = detailsText['languageCode'] - Identifier = location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC']) - - #Is this for a NWS Zone or County? - last4 = location[2:] - locationType = None - if 'C' in last4: - locationType = 'C' - elif 'Z' in last4: - locationType = 'Z' + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + theCode = await r.status + + #Set the actions based on response code + if theCode == 204: + l.info('No alerts for area ' + location + '.\n') + return + elif theCode == 403: + l.critical("Uh oh! Your API key may not be authorized for alerts. Tsk Tsk. Maybe you shouldn't pirate IBM data :)\n") + return + elif theCode == 401: + l.critical("Uh oh! This request requires authentication. Maybe you shouldn't try to access resources for IBM employee's only :)\n") + return + elif theCode == 404: + l.error("Uh oh! The requested resource cannot be found. This means either the URL is wrong or IBM is having technical difficulties :(\n Or.... They deleted the API :O\n") + return + elif theCode == 405: + l.error("Uh oh! Got a 405! This means that somehow.... someway..... this script made an invalid request. So sad..... So terrible..... :(\n") + return + elif theCode == 406: + l.critical("Uh oh! Got a 406! This means that IBM doesn't like us. :(\n") + return + elif theCode == 408: + l.error("Uh oh! We were too slow in providing IBM our alert request. Although I prefer to say we were Slowly Capable! :)\n") + return + elif theCode == 500: + l.error("Uh oh! Seems IBM's on call IT Tech spilled coffee on the server! Looks like no alerts for a while. Please check back later :)\n") + return + elif theCode == 502 or theCode == 503 or theCode == 504: + l.error("Uh oh! This is why you don't have interns messing with the server configuration. Please stand by while IBM's on call IT Tech resolves the issue :)\n") + return + elif theCode == 200: + pass - #theIdent = str(Identifier) - try: - thecheck = open('./.temp/alertmanifest.txt', "r") - check = thecheck.read() + # Map headline variables + l.debug('Found Alert for ' + location + '\n') + dataH = await r.json() + alertsRoot = dataH['alerts'] + + for x in alertsRoot: + detailKey = x['detailKey'] + #Lets get map our detail variables. + detailsUrl = 'https://api.weather.com/v3/alerts/detail?alertId=' + detailKey + '&format=json&language=en-US&apiKey=' + detailsApiKey + detailsResponse = requests.get(detailsUrl) + dataD = detailsResponse.json() + detailsRoot = dataD['alertDetail'] + theDetailsText = detailsRoot['texts'] + detailsText = theDetailsText[0] + descriptionRaw = detailsText['description'] + language = detailsText['languageCode'] + Identifier = location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC']) - if check.find(Identifier) != -1: - l.debug("Alert already sent...") - return - except FileNotFoundError: - l.warning("alert manifest does not exist (yet)") - k += 1 #We have an alert to send! + #Is this for a NWS Zone or County? + last4 = location[2:] + locationType = None + if 'C' in last4: + locationType = 'C' + elif 'Z' in last4: + locationType = 'Z' + + #theIdent = str(Identifier) + try: + async with aiofiles.open('./.temp/alertmanifest.txt', 'r' ) as checkFile: + c = await checkFile.read() + + if c.find(Identifier) != -1: + l.debug(f"{Identifier} was sent already, skipping..") + return + except FileNotFoundError: + l.warning("alert manifest does not exist (yet)") + + k += 1 #We have an alert to send! + + #Lets Map Our Vocal Codes! + vocalCheck = x['phenomena'] + '_' + x['significance'] + vocalCode = None - #Lets Map Our Vocal Codes! - vocalCheck = x['phenomena'] + '_' + x['significance'] - vocalCode = None - - if vocalCheck == 'HU_W': - vocalCode = 'HE001' - elif vocalCheck == 'TY_W': - vocalCode = 'HE002' - elif vocalCheck == 'HI_W': - vocalCode = 'HE003' - elif vocalCheck == 'TO_A': - vocalCode = 'HE004' - elif vocalCheck == 'SV_A': - vocalCode = 'HE005' - elif vocalCheck == 'HU_A': - vocalCode = 'HE006' - elif vocalCheck == 'TY_A': - vocalCode = 'HE007' - elif vocalCheck == 'TR_W': - vocalCode = 'HE008' - elif vocalCheck == 'TR_A': - vocalCode = 'HE009' - elif vocalCheck == 'TI_W': - vocalCode = 'HE010' - elif vocalCheck == 'HI_A': - vocalCode = 'HE011' - elif vocalCheck == 'TI_A': - vocalCode = 'HE012' - elif vocalCheck == 'BZ_W': - vocalCode = 'HE013' - elif vocalCheck == 'IS_W': - vocalCode = 'HE014' - elif vocalCheck == 'WS_W': - vocalCode = 'HE015' - elif vocalCheck == 'HW_W': - vocalCode = 'HE016' - elif vocalCheck == 'LE_W': - vocalCode = 'HE017' - elif vocalCheck == 'ZR_Y': - vocalCode = 'HE018' - elif vocalCheck == 'CF_W': - vocalCode = 'HE019' - elif vocalCheck == 'LS_W': - vocalCode = 'HE020' - elif vocalCheck == 'WW_Y': - vocalCode = 'HE021' - elif vocalCheck == 'LB_Y': - vocalCode = 'HE022' - elif vocalCheck == 'LE_Y': - vocalCode = 'HE023' - elif vocalCheck == 'BZ_A': - vocalCode = 'HE024' - elif vocalCheck == 'WS_A': - vocalCode = 'HE025' - elif vocalCheck == 'FF_A': - vocalCode = 'HE026' - elif vocalCheck == 'FA_A': - vocalCode = 'HE027' - elif vocalCheck == 'FA_Y': - vocalCode = 'HE028' - elif vocalCheck == 'HW_A': - vocalCode = 'HE029' - elif vocalCheck == 'LE_A': - vocalCode = 'HE030' - elif vocalCheck == 'SU_W': - vocalCode = 'HE031' - elif vocalCheck == 'LS_Y': - vocalCode = 'HE032' - elif vocalCheck == 'CF_A': - vocalCode = 'HE033' - elif vocalCheck == 'ZF_Y': - vocalCode = 'HE034' - elif vocalCheck == 'FG_Y': - vocalCode = 'HE035' - elif vocalCheck == 'SM_Y': - vocalCode = 'HE036' - elif vocalCheck == 'EC_W': - vocalCode = 'HE037' - elif vocalCheck == 'EH_W': - vocalCode = 'HE038' - elif vocalCheck == 'HZ_W': - vocalCode = 'HE039' - elif vocalCheck == 'FZ_W': - vocalCode = 'HE040' - elif vocalCheck == 'HT_Y': - vocalCode = 'HE041' - elif vocalCheck == 'WC_Y': - vocalCode = 'HE042' - elif vocalCheck == 'FR_Y': - vocalCode = 'HE043' - elif vocalCheck == 'EC_A': - vocalCode = 'HE044' - elif vocalCheck == 'EH_A': - vocalCode = 'HE045' - elif vocalCheck == 'HZ_A': - vocalCode = 'HE046' - elif vocalCheck == 'DS_W': - vocalCode = 'HE047' - elif vocalCheck == 'WI_Y': - vocalCode = 'HE048' - elif vocalCheck == 'SU_Y': - vocalCode = 'HE049' - elif vocalCheck == 'AS_Y': - vocalCode = 'HE050' - elif vocalCheck == 'WC_W': - vocalCode = 'HE051' - elif vocalCheck == 'FZ_A': - vocalCode = 'HE052' - elif vocalCheck == 'WC_A': - vocalCode = 'HE053' - elif vocalCheck == 'AF_W': - vocalCode = 'HE054' - elif vocalCheck == 'AF_Y': - vocalCode = 'HE055' - elif vocalCheck == 'DU_Y': - vocalCode = 'HE056' - elif vocalCheck == 'LW_Y': - vocalCode = 'HE057' - elif vocalCheck == 'LS_A': - vocalCode = 'HE058' - elif vocalCheck == 'HF_W': - vocalCode = 'HE059' - elif vocalCheck == 'SR_W': - vocalCode = 'HE060' - elif vocalCheck == 'GL_W': - vocalCode = 'HE061' - elif vocalCheck == 'HF_A': - vocalCode = 'HE062' - elif vocalCheck == 'UP_W': - vocalCode = 'HE063' - elif vocalCheck == 'SE_W': - vocalCode = 'HE064' - elif vocalCheck == 'SR_A': - vocalCode = 'HE065' - elif vocalCheck == 'GL_A': - vocalCode = 'HE066' - elif vocalCheck == 'MF_Y': - vocalCode = 'HE067' - elif vocalCheck == 'MS_Y': - vocalCode = 'HE068' - elif vocalCheck == 'SC_Y': - vocalCode = 'HE069' - elif vocalCheck == 'UP_Y': - vocalCode = 'HE073' - elif vocalCheck == 'LO_Y': - vocalCode = 'HE074' - elif vocalCheck == 'AF_V': - vocalCode = 'HE075' - elif vocalCheck == 'UP_A': - vocalCode = 'HE076' - elif vocalCheck == 'TAV_W': - vocalCode = 'HE077' - elif vocalCheck == 'TAV_A': - vocalCode = 'HE078' - elif vocalCheck == 'TO_W': - vocalCode = 'HE110' - else: - vocalCode = '' + if vocalCheck == 'HU_W': + vocalCode = 'HE001' + elif vocalCheck == 'TY_W': + vocalCode = 'HE002' + elif vocalCheck == 'HI_W': + vocalCode = 'HE003' + elif vocalCheck == 'TO_A': + vocalCode = 'HE004' + elif vocalCheck == 'SV_A': + vocalCode = 'HE005' + elif vocalCheck == 'HU_A': + vocalCode = 'HE006' + elif vocalCheck == 'TY_A': + vocalCode = 'HE007' + elif vocalCheck == 'TR_W': + vocalCode = 'HE008' + elif vocalCheck == 'TR_A': + vocalCode = 'HE009' + elif vocalCheck == 'TI_W': + vocalCode = 'HE010' + elif vocalCheck == 'HI_A': + vocalCode = 'HE011' + elif vocalCheck == 'TI_A': + vocalCode = 'HE012' + elif vocalCheck == 'BZ_W': + vocalCode = 'HE013' + elif vocalCheck == 'IS_W': + vocalCode = 'HE014' + elif vocalCheck == 'WS_W': + vocalCode = 'HE015' + elif vocalCheck == 'HW_W': + vocalCode = 'HE016' + elif vocalCheck == 'LE_W': + vocalCode = 'HE017' + elif vocalCheck == 'ZR_Y': + vocalCode = 'HE018' + elif vocalCheck == 'CF_W': + vocalCode = 'HE019' + elif vocalCheck == 'LS_W': + vocalCode = 'HE020' + elif vocalCheck == 'WW_Y': + vocalCode = 'HE021' + elif vocalCheck == 'LB_Y': + vocalCode = 'HE022' + elif vocalCheck == 'LE_Y': + vocalCode = 'HE023' + elif vocalCheck == 'BZ_A': + vocalCode = 'HE024' + elif vocalCheck == 'WS_A': + vocalCode = 'HE025' + elif vocalCheck == 'FF_A': + vocalCode = 'HE026' + elif vocalCheck == 'FA_A': + vocalCode = 'HE027' + elif vocalCheck == 'FA_Y': + vocalCode = 'HE028' + elif vocalCheck == 'HW_A': + vocalCode = 'HE029' + elif vocalCheck == 'LE_A': + vocalCode = 'HE030' + elif vocalCheck == 'SU_W': + vocalCode = 'HE031' + elif vocalCheck == 'LS_Y': + vocalCode = 'HE032' + elif vocalCheck == 'CF_A': + vocalCode = 'HE033' + elif vocalCheck == 'ZF_Y': + vocalCode = 'HE034' + elif vocalCheck == 'FG_Y': + vocalCode = 'HE035' + elif vocalCheck == 'SM_Y': + vocalCode = 'HE036' + elif vocalCheck == 'EC_W': + vocalCode = 'HE037' + elif vocalCheck == 'EH_W': + vocalCode = 'HE038' + elif vocalCheck == 'HZ_W': + vocalCode = 'HE039' + elif vocalCheck == 'FZ_W': + vocalCode = 'HE040' + elif vocalCheck == 'HT_Y': + vocalCode = 'HE041' + elif vocalCheck == 'WC_Y': + vocalCode = 'HE042' + elif vocalCheck == 'FR_Y': + vocalCode = 'HE043' + elif vocalCheck == 'EC_A': + vocalCode = 'HE044' + elif vocalCheck == 'EH_A': + vocalCode = 'HE045' + elif vocalCheck == 'HZ_A': + vocalCode = 'HE046' + elif vocalCheck == 'DS_W': + vocalCode = 'HE047' + elif vocalCheck == 'WI_Y': + vocalCode = 'HE048' + elif vocalCheck == 'SU_Y': + vocalCode = 'HE049' + elif vocalCheck == 'AS_Y': + vocalCode = 'HE050' + elif vocalCheck == 'WC_W': + vocalCode = 'HE051' + elif vocalCheck == 'FZ_A': + vocalCode = 'HE052' + elif vocalCheck == 'WC_A': + vocalCode = 'HE053' + elif vocalCheck == 'AF_W': + vocalCode = 'HE054' + elif vocalCheck == 'AF_Y': + vocalCode = 'HE055' + elif vocalCheck == 'DU_Y': + vocalCode = 'HE056' + elif vocalCheck == 'LW_Y': + vocalCode = 'HE057' + elif vocalCheck == 'LS_A': + vocalCode = 'HE058' + elif vocalCheck == 'HF_W': + vocalCode = 'HE059' + elif vocalCheck == 'SR_W': + vocalCode = 'HE060' + elif vocalCheck == 'GL_W': + vocalCode = 'HE061' + elif vocalCheck == 'HF_A': + vocalCode = 'HE062' + elif vocalCheck == 'UP_W': + vocalCode = 'HE063' + elif vocalCheck == 'SE_W': + vocalCode = 'HE064' + elif vocalCheck == 'SR_A': + vocalCode = 'HE065' + elif vocalCheck == 'GL_A': + vocalCode = 'HE066' + elif vocalCheck == 'MF_Y': + vocalCode = 'HE067' + elif vocalCheck == 'MS_Y': + vocalCode = 'HE068' + elif vocalCheck == 'SC_Y': + vocalCode = 'HE069' + elif vocalCheck == 'UP_Y': + vocalCode = 'HE073' + elif vocalCheck == 'LO_Y': + vocalCode = 'HE074' + elif vocalCheck == 'AF_V': + vocalCode = 'HE075' + elif vocalCheck == 'UP_A': + vocalCode = 'HE076' + elif vocalCheck == 'TAV_W': + vocalCode = 'HE077' + elif vocalCheck == 'TAV_A': + vocalCode = 'HE078' + elif vocalCheck == 'TO_W': + vocalCode = 'HE110' + else: + vocalCode = '' - #Do some date/time conversions - EndTimeUTCEpoch = x['expireTimeUTC'] - EndTimeUTC = datetime.utcfromtimestamp(EndTimeUTCEpoch).strftime('%Y%m%d%H%M') - #EndTimeUTC = EndTimeUTCString.astimezone(pytz.UTC) + #Do some date/time conversions + EndTimeUTCEpoch = x['expireTimeUTC'] + EndTimeUTC = datetime.utcfromtimestamp(EndTimeUTCEpoch).strftime('%Y%m%d%H%M') + #EndTimeUTC = EndTimeUTCString.astimezone(pytz.UTC) + + expireTimeEpoch = x['expireTimeUTC'] + expireTimeUTC = datetime.utcfromtimestamp(expireTimeEpoch).strftime('%Y%m%d%H%M') + + #V3 Alert API doesn't give us issueTime in UTC. So we have to convert ourselves. Ughhh!! + iTLDTS = x['issueTimeLocal'] + iTLDTO = datetime.strptime(iTLDTS, '%Y-%m-%dT%H:%M:%S%z') + issueTimeToUTC = iTLDTO.astimezone(pytz.UTC) + issueTimeUtc = issueTimeToUTC.strftime('%Y%m%d%H%M') + + processTimeEpoch = x['processTimeUTC'] + processTime = datetime.fromtimestamp(processTimeEpoch).strftime('%Y%m%d%H%M%S') + + #What is the action of this alert? + Action = None + if x['messageType'] == 'Update': + Action = 'CON' + elif x['messageType'] == 'New': + Action = 'NEW' - expireTimeEpoch = x['expireTimeUTC'] - expireTimeUTC = datetime.utcfromtimestamp(expireTimeEpoch).strftime('%Y%m%d%H%M') + #Fix description to replace new lines with space and add XML escape Chars. when needed + + description = ' '.join(descriptionRaw.splitlines()) + description = description.replace('&', '&') + description = description.replace('<', '<') + description = description.replace('>', '>') + description = description.replace('-', '') + description = description.replace(':', '') + + #Is this alert urgent? + urgency ='piss' + if vocalCheck == 'TO_W' or vocalCheck == 'SV_W' or vocalCheck == 'FF_W': + urgency = 'BEUrgent' + else: + urgency = 'BERecord' + + alertMsg = 'NOT_USED' + x['productIdentifier'] + 'NOT_USED' + Action + '' + x['officeCode'] + '' + x['phenomena'] + '' + x['significance'] + '' + x['eventTrackingNumber'] + '' + x['eventDescription'] + 'NOT_USED' + EndTimeUTC + '' + str(x['severityCode']) + 'NOT_USED' + expireTimeUTC + '' + location + '' + x['adminDistrictCode'] + 'NOT_USEDNOT_USEDNOT_USED' + x['identifier'] + '' + processTime + '' + issueTimeUtc + '' + x['headlineText'] + '' + vocalCode + 'NOT_USED' + description + 'NOT_USED' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '' - #V3 Alert API doesn't give us issueTime in UTC. So we have to convert ourselves. Ughhh!! - iTLDTS = x['issueTimeLocal'] - iTLDTO = datetime.strptime(iTLDTS, '%Y-%m-%dT%H:%M:%S%z') - issueTimeToUTC = iTLDTO.astimezone(pytz.UTC) - issueTimeUtc = issueTimeToUTC.strftime('%Y%m%d%H%M') - - processTimeEpoch = x['processTimeUTC'] - processTime = datetime.fromtimestamp(processTimeEpoch).strftime('%Y%m%d%H%M%S') - - #What is the action of this alert? - Action = None - if x['messageType'] == 'Update': - Action = 'CON' - elif x['messageType'] == 'New': - Action = 'NEW' - - #Fix description to replace new lines with space and add XML escape Chars. when needed - - description = ' '.join(descriptionRaw.splitlines()) - description = description.replace('&', '&') - description = description.replace('<', '<') - description = description.replace('>', '>') - description = description.replace('-', '') - description = description.replace(':', '') - - #Is this alert urgent? - urgency ='piss' - if vocalCheck == 'TO_W' or vocalCheck == 'SV_W' or vocalCheck == 'FF_W': - urgency = 'BEUrgent' - else: - urgency = 'BERecord' - - alertMsg = 'NOT_USED' + x['productIdentifier'] + 'NOT_USED' + Action + '' + x['officeCode'] + '' + x['phenomena'] + '' + x['significance'] + '' + x['eventTrackingNumber'] + '' + x['eventDescription'] + 'NOT_USED' + EndTimeUTC + '' + str(x['severityCode']) + 'NOT_USED' + expireTimeUTC + '' + location + '' + x['adminDistrictCode'] + 'NOT_USEDNOT_USEDNOT_USED' + x['identifier'] + '' + processTime + '' + issueTimeUtc + '' + x['headlineText'] + '' + vocalCode + 'NOT_USED' + description + 'NOT_USED' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '' - - #Append BERecord - with open('./.temp/BERecord.xml', "a") as b: - b.write(alertMsg) - b.close() - - #Add our alert to the manifest so we don't keep sending in the same alert every 60 seconds unless an update is issued. - with open('./.temp/alertmanifest.txt', "a") as c: - c.write('\n' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC'])) - c.close() + #Append BERecord + async with aiofiles.open('./.temp/BERecord.xml', "a") as b: + await b.write(alertMsg) + await b.close() + + #Add our alert to the manifest so we don't keep sending in the same alert every 60 seconds unless an update is issued. + async with aiofiles.open('./.temp/alertmanifest.txt', "a") as c: + await c.write('\n' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC'])) + await c.close() -# TODO: This should be converted into a function so it works better with async, that way we're not getting hung up on that time.sleep() call. -def makeRecord(): +async def makeRecord(): + loop = asyncio.get_running_loop() global k - with open("./.temp/BERecord.xml", 'a') as BERecord: - BERecord.write('') - BERecord.close() - - for z in alertLocations: - getAlerts(z) - with open('./.temp/BERecord.xml', 'a') as BERecord: - BERecord.write("") - BERecord.close() - - dom = xml.dom.minidom.parse("./.temp/BERecord.xml") - pretty_xml_as_string = dom.toprettyxml(indent = " ") - - with open("./.temp/BERecord.i2m", 'w') as h: - h.write(pretty_xml_as_string[23:]) - h.close() - - # If we don't need to send the i2 an alert, we don't need to gzip it. + # The BERecord XML doesn't need to be written if there's no alerts. if k > 0: + async with aiofiles.open("./.temp/BERecord.xml", 'a') as BERecord: + await BERecord.write('') + await BERecord.close() + + for z in alertLocations: + await getAlerts(z) + + async with aiofiles.open('./.temp/BERecord.xml', 'a') as BERecord: + await BERecord.write("") + await BERecord.close() + + dom = xml.dom.minidom.parse("./.temp/BERecord.xml") + pretty_xml_as_string = dom.toprettyxml(indent = " ") + + async with aiofiles.open("./.temp/BERecord.i2m", 'w') as h: + await h.write(pretty_xml_as_string[23:]) + await h.close() + l.info("Sending alert(s) to the IntelliStar 2!") with open("./.temp/BERecord.i2m", 'rb') as f_in: with gzip.open("./.temp/BERecord.gz", 'wb') as f_out: @@ -354,7 +360,7 @@ def makeRecord(): bit.sendFile(files, commands, 1, 0) os.remove(gZipFile) k = 0 - - os.remove("./.temp/BERecord.xml") - os.remove("./.temp/BERecord.i2m") + + os.remove("./.temp/BERecord.xml") + os.remove("./.temp/BERecord.i2m") \ No newline at end of file diff --git a/recordGenerators/Breathing.py b/recordGenerators/Breathing.py index 6d93690..991c85b 100644 --- a/recordGenerators/Breathing.py +++ b/recordGenerators/Breathing.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles, asyncio sys.path.append("./py2lib") sys.path.append("./Util") @@ -30,14 +31,14 @@ l.debug(coopIds, geocodes) apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/breathing/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" + data = "" #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[63:-26] @@ -45,32 +46,33 @@ def getData(coopId, geocode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(coopId) + '' - f = open("./.temp/Breathing.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/Breathing.i2m", "a") as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): +async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a Breathing forecast record.") header = '' footer = '' - with open("./.temp/Breathing.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/Breathing.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(coopIds, geocodes): - getData(x, y) + await getData(x, y) - with open("./.temp/Breathing.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/Breathing.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/Breathing.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/Breathing.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/Breathing.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index d33c012..b41da7a 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles, asyncio import sys sys.path.append("./py2lib") @@ -34,50 +35,53 @@ for i in MPC.getMetroCities(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(tecci, zipCode): +async def getData(tecci, zipCode): + l.debug('Gathering data for location id ' + tecci) fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/observations/current.xml?language=en-US&units=e&apiKey=' + apiKey + data = "" - #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[67:-30] - l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' + async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as f: + await f.write(i2Doc) + await f.close() - f = open("./.temp/CurrentObservations.i2m", "a") - f.write(i2Doc) - f.close() - -def makeDataFile(): +async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a CurrentObservations record.") header = '' footer = '' - with open("./.temp/CurrentObservations.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/CurrentObservations.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(tecciId, zipCodes): - getData(x, y) + await getData(x, y) - with open("./.temp/CurrentObservations.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/CurrentObservations.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/CurrentObservations.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/CurrentObservations.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] + """ + TODO: This can be ran in a seperate thread using loop.run_in_executor() according to the python discord. + ! This should probably be implemented ASAP. + """ with open("./.temp/CurrentObservations.i2m", 'rb') as f_in: with gzip.open("./.temp/CurrentObservations.gz", 'wb') as f_out: shutil.copyfileobj(f_in, f_out) @@ -88,7 +92,7 @@ def makeDataFile(): command = commands.append('') numFiles = len(files) - bit.sendFile(files, commands, numFiles, 0) + await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0)) os.remove("./.temp/CurrentObservations.i2m") os.remove("./.temp/CurrentObservations.gz") diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index acd7133..5becbfe 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles, asyncio sys.path.append("./py2lib") sys.path.append("./Util") @@ -32,14 +33,13 @@ for i in MPC.getMetroCities(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(tecci, zipCode): +async def getData(tecci, zipCode): fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/daily/7day.xml?language=en-US&units=e&apiKey=' + apiKey + data = "" - #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[61:-24] @@ -47,32 +47,33 @@ def getData(tecci, zipCode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' - f = open("./.temp/DailyForecast.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/DailyForecast.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): +async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a DailyForecast record.") header = '' footer = '' - with open("./.temp/DailyForecast.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/DailyForecast.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(tecciId, zipCodes): - getData(x, y) + await getData(x, y) - with open("./.temp/DailyForecast.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/DailyForecast.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/DailyForecast.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/DailyForecast.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/DailyForecast.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] diff --git a/recordGenerators/HeatingAndCooling.py b/recordGenerators/HeatingAndCooling.py index 214fff7..e2790bb 100644 --- a/recordGenerators/HeatingAndCooling.py +++ b/recordGenerators/HeatingAndCooling.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,49 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/heatCool/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" - - res = requests.get(fetchUrl) - - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write HeatingAndCooling record -- Status code {res.status_code}") - return + data = "" - data = res.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write HeatingAndCooling record -- Status code {r.status}") + return + + data = await r.text() + + # data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/HeatingAndCooling.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing HeatingAndCooling record.") header = '' footer = '' - with open('./.temp/HeatingAndCooling.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/HeatingAndCooling.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/HeatingAndCooling.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/HeatingAndCooling.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index 1e6b097..d1ea73e 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -5,6 +5,7 @@ import os import shutil import xml.dom.minidom import logging,coloredlogs +import aiohttp, aiofiles, asyncio, asyncio import sys sys.path.append("./py2lib") @@ -32,46 +33,49 @@ for i in MPC.getMetroCities(): apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(tecci, zipCode): +async def getData(tecci, zipCode): + l.debug('Gathering data for location id ' + tecci) fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/hourly/360hour.xml?language=en-US&units=e&apiKey=' + apiKey + data = "" #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[48:-11] - l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' - f = open("./.temp/HourlyForecast.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/HourlyForecast.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): + +async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing an HourlyForecast record.") header = '' footer = '' - with open("./.temp/HourlyForecast.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/HourlyForecast.i2m", 'w') as doc: + await doc.write(header) + for x, y in zip(tecciId, zipCodes): - getData(x, y) + await getData(x, y) - with open("./.temp/HourlyForecast.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/HourlyForecast.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/HourlyForecast.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/HourlyForecast.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/HourlyForecast.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] diff --git a/recordGenerators/MosquitoActivity.py b/recordGenerators/MosquitoActivity.py index 0486dc3..dba9acd 100644 --- a/recordGenerators/MosquitoActivity.py +++ b/recordGenerators/MosquitoActivity.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,49 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/mosquito/daily/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" + data = "" + + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write MosquitoActivity record -- status code {r.status}") + return + + data = await r.text() - res = requests.get(fetchUrl) - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write MosquitoActivity record -- Status code {res.status_code}") - return - - data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/MosquitoActivity.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing MosquitoActivity record.") header = '' footer = '' - with open('./.temp/MosquitoActivity.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/MosquitoActivity.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/MosquitoActivity.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/MosquitoActivity.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip diff --git a/recordGenerators/PollenForecast.py b/recordGenerators/PollenForecast.py index 11a8eed..960b2e1 100644 --- a/recordGenerators/PollenForecast.py +++ b/recordGenerators/PollenForecast.py @@ -6,6 +6,7 @@ import os import shutil import xml.dom.minidom import logging, coloredlogs +import aiohttp, aiofiles, asyncio sys.path.append("./py2lib") sys.path.append("./Util") @@ -31,14 +32,13 @@ l.debug(pollenIds, geocodes) apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' -def getData(pollenId, geocode): +async def getData(pollenId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/pollen/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" - + data = "" #Fetch data - - response = requests.get(fetchUrl) - - data = response.text + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + data = await r.text() newData = data[63:-26] @@ -46,32 +46,33 @@ def getData(pollenId, geocode): #Write to .i2m file i2Doc = '' + '' + newData + '' + str(pollenId) + '' - f = open("./.temp/PollenForecast.i2m", "a") - f.write(i2Doc) - f.close() + async with aiofiles.open("./.temp/PollenForecast.i2m", "a") as f: + await f.write(i2Doc) + await f.close() -def makeDataFile(): +async def makeDataFile(): + loop = asyncio.get_running_loop() l.info("Writing a PollenForecast record.") header = '' footer = '' - with open("./.temp/PollenForecast.i2m", 'w') as doc: - doc.write(header) + async with aiofiles.open("./.temp/PollenForecast.i2m", 'w') as doc: + await doc.write(header) for x, y in zip(pollenIds, geocodes): - getData(x, y) + await getData(x, y) - with open("./.temp/PollenForecast.i2m", 'a') as end: - end.write(footer) + async with aiofiles.open("./.temp/PollenForecast.i2m", 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse("./.temp/PollenForecast.i2m") pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open("./.temp/PollenForecast.i2m", "w") as g: - g.write(pretty_xml_as_string[23:]) - g.close() + async with aiofiles.open("./.temp/PollenForecast.i2m", "w") as g: + await g.write(pretty_xml_as_string[23:]) + await g.close() files = [] commands = [] diff --git a/recordGenerators/TideForecast.py b/recordGenerators/TideForecast.py index 72d535a..3dce2a7 100644 --- a/recordGenerators/TideForecast.py +++ b/recordGenerators/TideForecast.py @@ -1,6 +1,4 @@ import shutil -from xmlrpc.client import DateTime -import requests import logging,coloredlogs import datetime from py2Lib import bit @@ -9,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -22,31 +21,34 @@ for i in MPC.getTideStations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(tideStation, geocode): +async def getData(tideStation, geocode): today = datetime.date.today() startDate = today.strftime('%Y%m%d') endDate_unformatted = datetime.datetime.strptime(startDate, '%Y%m%d') + datetime.timedelta(days=5) endDate = endDate_unformatted.strftime('%Y%m%d') + data = "" fetchUrl = f"https://api.weather.com/v1/geocode/{geocode}/forecast/tides.xml?language=en-US&units=e&startDate={startDate}&endDate={endDate}&apiKey={apiKey}" - res = requests.get(fetchUrl) + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to write TideForecast -- status code {r.status}") + return + + data = await r.text() + - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write TidesForecast record -- Status code {res.status_code}") - return - - data = res.text newData = data[53:-16] i2Doc = f'\n \n {newData}\n {tideStation}\n ' - f = open('./.temp/TidesForecast.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): + loop = asyncio.get_running_loop() if len(tideStations) < 1: l.debug("Skipping TidesForecast -- No locations.") return @@ -56,21 +58,21 @@ def makeRecord(): header = '' footer = '' - with open('./.temp/TidesForecast.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(tideStations, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/TidesForecast.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/TidesForecast.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/TidesForecast.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/TidesForecast.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip diff --git a/recordGenerators/WateringNeeds.py b/recordGenerators/WateringNeeds.py index 0436811..7a3c494 100644 --- a/recordGenerators/WateringNeeds.py +++ b/recordGenerators/WateringNeeds.py @@ -7,6 +7,7 @@ import records.LFRecord as LFR import gzip from os import remove import xml.dom.minidom +import aiohttp, aiofiles, asyncio l = logging.getLogger(__name__) coloredlogs.install() @@ -20,46 +21,48 @@ for i in MPC.getPrimaryLocations(): apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" -def getData(coopId, geocode): +async def getData(coopId, geocode): fetchUrl = f"https://api.weather.com/v2/indices/wateringNeeds/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" - - res = requests.get(fetchUrl) - - if res.status_code != 200: - l.error("DO NOT REPORT THE ERROR BELOW") - l.error(f"Failed to write WateringNeeds record -- Status code {res.status_code}") - return + data = "" + + async with aiohttp.ClientSession() as s: + async with s.get(fetchUrl) as r: + if r.status != 200: + l.error(f"Failed to WateringNeeds -- status code {r.status}") + return + + data = await r.text() - data = res.text newData = data[63:-26] i2Doc = f'\n \n {newData}\n {coopId}\n ' - f = open('./.temp/WateringNeeds.i2m', 'a') - f.write(i2Doc) - f.close() + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as f: + await f.write(i2Doc) + await f.close() -def makeRecord(): +async def makeRecord(): + loop = asyncio.get_running_loop() l.info("Writing WateringNeeds record.") header = '' footer = '' - with open('./.temp/WateringNeeds.i2m', 'a') as doc: - doc.write(header) + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as doc: + await doc.write(header) for (x, y) in zip(coopIds, geocodes): - getData(x,y) + await getData(x,y) - with open('./.temp/WateringNeeds.i2m', 'a') as end: - end.write(footer) + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as end: + await end.write(footer) dom = xml.dom.minidom.parse('./.temp/WateringNeeds.i2m') xmlPretty = dom.toprettyxml(indent= " ") - with open('./.temp/WateringNeeds.i2m', 'w') as g: - g.write(xmlPretty[23:]) - g.close() + async with aiofiles.open('./.temp/WateringNeeds.i2m', 'w') as g: + await g.write(xmlPretty[23:]) + await g.close() # Compresss i2m to gzip diff --git a/recordGenerators/__init__.py b/recordGenerators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/records/LFRecord.py b/records/LFRecord.py index 330b3b0..cd15b95 100644 --- a/records/LFRecord.py +++ b/records/LFRecord.py @@ -37,8 +37,4 @@ def getLatLong(locId: str): return fetched[0] + "/" + fetched[1] def getLocationInfo(locId: str): - pass - -print(getCoopId('USAZ0278')) -print(getZip('USAZ0278')) -print(getLatLong('USAZ0278')) \ No newline at end of file + pass \ No newline at end of file