mirror/i2ME-Legacy
1
0
Fork 0
mirror of https://github.com/mewtek/i2ME-Legacy.git synced 2025-05-14 18:21:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6 from Open-Telecom/collectorsToAsync

Browse Source 
Convert data collectors to async
This commit is contained in:
April Wormwood 2022-11-13 17:19:56 -07:00 committed by GitHub
commit 66a769fe9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 828 additions and 624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
RecordTasks.py Normal file
View File

@ -0,0 +1,97 @@
import asyncio
from recordGenerators import Alerts,CurrentObservations,HourlyForecast,DailyForecast, AirQuality, AirportDelays, AchesAndPains, Breathing, HeatingAndCooling, MosquitoActivity, PollenForecast, TideForecast, WateringNeeds
from radar import TWCRadarCollector
from datetime import datetime
""" This houses the tasks needed to update the data records concurrently
I have no idea if this is a messy way to do things, but it will be worked upon if it is.
"""
async def updateMosaicTask():
mosaicUpdateIntervals = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
while True:
# Server takes ~15 - 35 seconds to fully generate a frame, so use 40 seconds to ensure it's fully generated.
if datetime.now().minute in mosaicUpdateIntervals and datetime.now().second == 40:
await TWCRadarCollector.collect("radarmosaic")
await asyncio.sleep(1)
async def updateSatradTask():
satradUpdateIntervals = [0, 10, 20, 30, 40, 50]
while True:
# Server takes ~15 - 35 seconds to fully generate a frame, so use 40 seconds to ensure it's fully generated.
if datetime.now().minute in satradUpdateIntervals and datetime.now().second == 40:
await TWCRadarCollector.collect("satrad")
await asyncio.sleep(1)
async def alertsTask():
while True:
await Alerts.makeRecord()
await asyncio.sleep(60)
async def coTask():
while True:
await CurrentObservations.makeDataFile()
await asyncio.sleep(5 * 60)
# These tasks should be updated every hour
async def hfTask():
while True:
await HourlyForecast.makeDataFile()
await asyncio.sleep(60 * 60)
async def dfTask():
while True:
await DailyForecast.makeDataFile()
await asyncio.sleep(60 * 60)
async def aqTask():
while True:
await AirQuality.writeData()
await asyncio.sleep(60 * 60)
async def aptTask():
while True:
await AirportDelays.writeData()
await asyncio.sleep(60 * 60)
async def apTask():
while True:
await AchesAndPains.makeRecord()
await asyncio.sleep(60 * 60)
async def brTask():
while True:
await Breathing.makeDataFile()
await asyncio.sleep(60 * 60)
async def hcTask():
while True:
await HeatingAndCooling.makeRecord()
await asyncio.sleep(60 * 60)
async def maTask():
while True:
await MosquitoActivity.makeRecord()
await asyncio.sleep(60 * 60)
async def pTask():
while True:
await PollenForecast.makeDataFile()
await asyncio.sleep(60 * 60)
async def tTask():
while True:
await TideForecast.makeRecord()
await asyncio.sleep(60 * 60)
async def wnTask():
while True:
await WateringNeeds.makeRecord()
await asyncio.sleep(60 * 60)

85
Util/MachineProductCfg.py
View File

@ -50,9 +50,90 @@ def getTideStations():
def getAirportCodes(): def getAirportCodes():
""" Returns all of the airport identifiers present in the MachineProductCfg """ """ Returns all of the airport identifiers present in the MachineProductCfg """
airports = [] airports = [
'ATL',
'LAX',
'ORD',
'DFW',
'JFK',
'DEN',
'SFO',
'CLT',
'LAS',
'PHX',
'IAH',
'MIA',
'SEA',
'EWR',
'MCO',
'MSP',
'DTW',
'BOS',
'PHL',
'LGA',
'FLL',
'BWI',
'IAD',
'MDW',
'SLC',
'DCA',
'HNL',
'SAN',
'TPA',
'PDX',
'STL',
'HOU',
'BNA',
'AUS',
'OAK',
'MSY',
'RDU',
'SJC',
'SNA',
'DAL',
'SMF',
'SAT',
'RSW',
'PIT',
'CLE',
'IND',
'MKE',
'CMH',
'OGG',
'PBI',
'BDL',
'CVG',
'JAX',
'ANC',
'BUF',
'ABQ',
'ONT',
'OMA',
'BUR',
'OKC',
'MEM',
'PVD',
'RIC',
'SDF',
'RNO',
'TUS',
'CHS',
'ORF',
'PWM',
'GRR',
'BHM',
'LIT',
'DSM',
'FAR',
'FSD',
'ICT',
'LBB',
'BIL',
'BOI',
'GEG'
]
for i in data['Config']['ConfigDef']['ConfigItems']['ConfigItem']: for i in data['Config']['ConfigDef']['ConfigItems']['ConfigItem']:
if "Airport" in i['@key'] and i['@value'] != "": if "Airport" in i['@key'] and i['@value'] != "" and not i['@value'] in airports:
# Split the string up # Split the string up
airports.append(i['@value'].split("_")[2]) airports.append(i['@value'].split("_")[2])

126
main.py
View File

@ -1,93 +1,81 @@
import asyncio import asyncio, aiofiles
from asyncio.log import logger from asyncio.log import logger
from asyncore import loop from asyncore import loop
import logging,coloredlogs import logging,coloredlogs
from recordGenerators import Alerts,CurrentObservations,DailyForecast,HourlyForecast,AirportDelays,AirQuality,HeatingAndCooling,PollenForecast,Breathing, AchesAndPains, MosquitoActivity, WateringNeeds, TideForecast
from radar import TWCRadarCollector from radar import TWCRadarCollector
import os import os
from datetime import datetime from datetime import datetime
import RecordTasks
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install(logger=l) coloredlogs.install(logger=l)
useRadarServer = True useRadarServer = True
# Create dirs and files
if not os.path.exists('.temp/'):
os.makedirs('.temp/')
if not os.path.exists('.temp/tiles/'):
os.makedirs('.temp/tiles/')
if not os.path.exists('.temp/tiles/output/'):
os.makedirs('.temp/tiles/output/')
if not os.path.exists('.temp/msgId.txt'):
print("Creating initial msgId file")
with open('.temp/msgId.txt', "w") as f:
f.write("410080515")
"""
CurrentConditions: Every 5 minutes
Daily Forecasts, Hourlies, etc: 60 minutes
Alerts: 5 minutes
"""
l.info("Starting i2RecordCollector") l.info("Starting i2RecordCollector")
l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose") l.info("Developed by mewtek32, Floppaa, Goldblaze, and needlenose")
async def grabAlertsLoop(): async def createTemp():
while True: """ Used on a first time run, creates necessary files & directories for the message encoder to work properly. """
Alerts.makeRecord() if not (os.path.exists('./.temp/')):
await asyncio.sleep(60) l.info("Creating necessary directories & files..")
os.mkdir('./.temp')
async def FiveMinUpdaters(): # Used for the record generator
while True: os.mkdir('./.temp/tiles/')
CurrentObservations.makeDataFile() os.mkdir('./.temp/tiles/output/')
l.debug("Sleeping for 5 minutes...")
await asyncio.sleep(5 * 60)
async def HourUpdaters(): # Used for radar server downloads
while True: os.mkdir('./.temp/output')
DailyForecast.makeDataFile() os.mkdir('./.temp/output/radarmosaic')
HourlyForecast.makeDataFile() os.mkdir('./.temp/output/satrad')
AirQuality.writeData()
PollenForecast.makeDataFile()
AirportDelays.writeData()
Breathing.makeDataFile()
HeatingAndCooling.makeRecord()
WateringNeeds.makeRecord()
MosquitoActivity.makeRecord()
AchesAndPains.makeRecord()
TideForecast.makeRecord()
l.debug("Sleeping for an hour...")
await asyncio.sleep(60 * 60)
async def radarCollector(): # Create msgId file for bit.py
mosaicUpdateIntervals = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] async with aiofiles.open('./.temp/msgId.txt', 'w') as msgId:
satradUpdateIntervals = [0, 10, 20, 30, 40, 50] await msgId.write('410080515')
await msgId.close()
else:
l.debug(".temp file exists")
return
while True:
# Server takes ~15 - 35 seconds on average to fully generate a frame, use 40 seconds
# to make sure the radar frame is fully good to go
if datetime.now().minute in mosaicUpdateIntervals and datetime.now().second == 40:
await TWCRadarCollector.collect("radarmosaic")
if datetime.now().minute in satradUpdateIntervals and datetime.now().second == 45: async def main():
await TWCRadarCollector.collect("satrad") await createTemp()
await asyncio.sleep(1) mosaicTask = asyncio.create_task(RecordTasks.updateMosaicTask())
satradTask = asyncio.create_task(RecordTasks.updateSatradTask())
alertsTask = asyncio.create_task(RecordTasks.alertsTask())
coTask = asyncio.create_task(RecordTasks.coTask())
hfTask = asyncio.create_task(RecordTasks.hfTask())
dfTask = asyncio.create_task(RecordTasks.dfTask())
aqTask = asyncio.create_task(RecordTasks.aqTask())
aptTask = asyncio.create_task(RecordTasks.aptTask())
apTask = asyncio.create_task(RecordTasks.apTask())
brTask = asyncio.create_task(RecordTasks.brTask())
hcTask = asyncio.create_task(RecordTasks.hcTask())
maTask = asyncio.create_task(RecordTasks.maTask())
pTask = asyncio.create_task(RecordTasks.pTask())
tTask = asyncio.create_task(RecordTasks.tTask())
wnTask = asyncio.create_task(RecordTasks.wnTask())
loop = asyncio.get_event_loop() # In theory, these should all run concurrently without problems
alertTask = loop.create_task(grabAlertsLoop()) await alertsTask
CCtask = loop.create_task(FiveMinUpdaters()) await coTask
ForecastsTask = loop.create_task(HourUpdaters()) await hfTask
await dfTask
await aqTask
await aptTask
await apTask
await brTask
await hcTask
await maTask
await pTask
await tTask
await wnTask
if useRadarServer: radarTask = loop.create_task(radarCollector()) if useRadarServer:
await mosaicTask
await satradTask
try: if __name__ == "__main__":
loop.run_until_complete(alertTask) asyncio.run(main())
loop.run_until_complete(CCtask)
loop.run_until_complete(ForecastsTask)
if useRadarServer: loop.run_until_complete(radarTask)
except asyncio.CancelledError: pass

2
py2Lib/starbundle.py
View File

@ -42,4 +42,4 @@ def makeStarBundle(Directory, Type, flag, Version, date, sendAfter):
#Directory which contains Files to be bundled Type Flags Version Date SendImmediately(Does not apply to this script) #Directory which contains Files to be bundled Type Flags Version Date SendImmediately(Does not apply to this script)
makeStarBundle('./.temp/i2State/SD/Managed/Events', 'Managed', 'Domestic_SD_Universe', '637898877227230030', '09/28/2022', 0) # makeStarBundle('./.temp/i2State/SD/Managed/Events', 'Managed', 'Domestic_SD_Universe', '637898877227230030', '09/28/2022', 0)

6
radar/RadarProcessor.py
View File

@ -55,8 +55,8 @@ def WorldCoordinateToTile(coord: Point) -> Point:
scale = 1 << 6 scale = 1 << 6
return Point( return Point(
x = math.floor(coord.x * scale / 256), x = math.floor(coord.x * scale / 255),
y = math.floor(coord.y * scale / 256) y = math.floor(coord.y * scale / 255)
) )
def WorldCoordinateToPixel(coord: Point) -> Point: def WorldCoordinateToPixel(coord: Point) -> Point:
@ -68,7 +68,7 @@ def WorldCoordinateToPixel(coord: Point) -> Point:
) )
def LatLongProject(lat, long) -> Point: def LatLongProject(lat, long) -> Point:
siny = math.sin(lat * math.pi / 185) siny = math.sin(lat * math.pi / 180)
siny = min(max(siny, -0.9999), 0.9999) siny = min(max(siny, -0.9999), 0.9999)
return Point( return Point(

1
radar/TWCRadarCollector.py
View File

@ -88,6 +88,7 @@ def getTime(timestamp) -> str:
return str(time) return str(time)
async def collect(radarType: str): async def collect(radarType: str):
loop = asyncio.get_running_loop()
ts = await getValidTimestamps(radarType) ts = await getValidTimestamps(radarType)
frames = await downloadRadarFrames(radarType, ts) frames = await downloadRadarFrames(radarType, ts)

42
recordGenerators/AchesAndPains.py
View File

@ -7,6 +7,7 @@ import records.LFRecord as LFR
import gzip import gzip
from os import remove from os import remove
import xml.dom.minidom import xml.dom.minidom
import aiohttp, aiofiles, asyncio
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install() coloredlogs.install()
@ -20,46 +21,49 @@ for i in MPC.getPrimaryLocations():
apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" apiKey = "21d8a80b3d6b444998a80b3d6b1449d3"
def getData(coopId, geocode): async def getData(coopId, geocode):
fetchUrl = f"https://api.weather.com/v2/indices/achePain/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v2/indices/achePain/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}"
data = ""
res = requests.get(fetchUrl) async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
if res.status_code != 200: if r.status != 200:
l.error("DO NOT REPORT THE ERROR BELOW") l.error(f"Failed to write AchesAndPains record -- status code {r.status}")
l.error(f"Failed to write AchesAndPains record -- Status code {res.status_code}")
return return
data = res.text data = await r.text()
newData = data[63:-26] newData = data[63:-26]
i2Doc = f'\n <AchesAndPains id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </AchesAndPains>' i2Doc = f'\n <AchesAndPains id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </AchesAndPains>'
f = open('./.temp/AchesAndPains.i2m', 'a') async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeRecord(): async def makeRecord():
loop = asyncio.get_running_loop()
l.info("Writing AchesAndPains record.") l.info("Writing AchesAndPains record.")
header = '<Data type="AchesAndPains">' header = '<Data type="AchesAndPains">'
footer = '</Data>' footer = '</Data>'
with open('./.temp/AchesAndPains.i2m', 'a') as doc: async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as doc:
doc.write(header) await doc.write(header)
for (x, y) in zip(coopIds, geocodes): for (x, y) in zip(coopIds, geocodes):
getData(x,y) await getData(x,y)
with open('./.temp/AchesAndPains.i2m', 'a') as end: async with aiofiles.open('./.temp/AchesAndPains.i2m', 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse('./.temp/AchesAndPains.i2m') dom = xml.dom.minidom.parse('./.temp/AchesAndPains.i2m')
xmlPretty = dom.toprettyxml(indent= " ") xmlPretty = dom.toprettyxml(indent= " ")
with open('./.temp/AchesAndPains.i2m', 'w') as g: async with aiofiles.open('./.temp/AchesAndPains.i2m', 'w') as g:
g.write(xmlPretty[23:]) await g.write(xmlPretty[23:])
g.close() await g.close()
# Compresss i2m to gzip # Compresss i2m to gzip

34
recordGenerators/AirQuality.py
View File

@ -4,6 +4,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install() coloredlogs.install()
@ -27,22 +28,25 @@ for i in MPC.getPrimaryLocations():
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(epaId, zipcode): async def getData(epaId, zipcode):
url = f"https://api.weather.com/v1/location/{zipcode}:4:US/airquality.xml?language=en-US&apiKey={apiKey}" url = f"https://api.weather.com/v1/location/{zipcode}:4:US/airquality.xml?language=en-US&apiKey={apiKey}"
data = ""
res = requests.get(url=url) async with aiohttp.ClientSession() as s:
async with s.get(url) as r:
data = await r.text()
data = res.text
newData = data[57:-11] newData = data[57:-11]
# Write to i2doc file # Write to i2doc file
i2Doc = f'<AirQuality id="000000000" locationKey="{epaId}" isWxScan="0">' + '' + newData + f'<clientKey>{epaId}</clientKey></AirQuality>' i2Doc = f'<AirQuality id="000000000" locationKey="{epaId}" isWxScan="0">' + '' + newData + f'<clientKey>{epaId}</clientKey></AirQuality>'
f = open("./.temp/AirQuality.i2m", 'a') async with aiofiles.open("./.temp/AirQuality.i2m", 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def writeData(): async def writeData():
loop = asyncio.get_running_loop()
useData = False useData = False
workingEpaIds = [] workingEpaIds = []
@ -62,21 +66,21 @@ def writeData():
header = '<Data type="AirQuality">' header = '<Data type="AirQuality">'
footer = "</Data>" footer = "</Data>"
with open("./.temp/AirQuality.i2m", 'w') as doc: async with aiofiles.open("./.temp/AirQuality.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for (x, y) in zip(workingEpaIds, zipCodes): for (x, y) in zip(workingEpaIds, zipCodes):
getData(x, y) await getData(x, y)
with open("./.temp/AirQuality.i2m", 'a') as end: async with aiofiles.open("./.temp/AirQuality.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/AirQuality.i2m") dom = xml.dom.minidom.parse("./.temp/AirQuality.i2m")
xmlPretty = dom.toprettyxml(indent = " ") xmlPretty = dom.toprettyxml(indent = " ")
with open("./.temp/AirQuality.i2m", 'w') as g: async with aiofiles.open("./.temp/AirQuality.i2m", 'w') as g:
g.write(xmlPretty[23:]) await g.write(xmlPretty[23:])
g.close() await g.close()
files = [] files = []
commands = [] commands = []

46
recordGenerators/AirportDelays.py
View File

@ -4,6 +4,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio
import sys import sys
sys.path.append("./py2lib") sys.path.append("./py2lib")
@ -29,34 +30,35 @@ l.debug(airports)
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(airport): async def getData(airport):
url = f"https://api.weather.com/v1/airportcode/{airport}/airport/delays.xml?language=en-US&apiKey={apiKey}" url = f"https://api.weather.com/v1/airportcode/{airport}/airport/delays.xml?language=en-US&apiKey={apiKey}"
data = ""
res = requests.get(url=url) async with aiohttp.ClientSession() as s:
async with s.get(url) as r:
data = await r.text()
data = res.text newData = data[48:-11].replace('¿', '-')
newData = data[48:-11]
# Write to i2doc file # Write to i2doc file
i2Doc = f'<AirportDelays id="000000000" locationKey="{airport}" isWxScan="0">' + '' + newData + f'<clientKey>{airport}</clientKey></AirportDelays>' i2Doc = f'<AirportDelays id="000000000" locationKey="{airport}" isWxScan="0">' + '' + newData + f'<clientKey>{airport}</clientKey></AirportDelays>'
f = open("./.temp/AirportDelays.i2m", 'a') async with aiofiles.open("./.temp/AirportDelays.i2m", 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def writeData(): async def writeData():
loop = asyncio.get_running_loop()
useData = False useData = False
airportsWithDelays = [] airportsWithDelays = []
for x in airports: for x in airports:
# Do a quick check to see if the airport in question has a delay or not async with aiohttp.ClientSession() as s:
res = requests.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") async with s.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") as r:
if r.status != 200:
if (res.status_code != 200): l.debug(f"No delay for {x} found, skipping..")
l.debug(f"[AIRPORT DELAYS] No delays for {x} found, skipping..")
else: else:
airportsWithDelays.append(x) airportsWithDelays.append(x)
l.debug(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..")
useData = True useData = True
if (useData): if (useData):
@ -64,21 +66,21 @@ def writeData():
header = '<Data type="AirportDelays">' header = '<Data type="AirportDelays">'
footer = "</Data>" footer = "</Data>"
with open("./.temp/AirportDelays.i2m", 'w') as doc: async with aiofiles.open("./.temp/AirportDelays.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for x in airportsWithDelays: for x in airportsWithDelays:
getData(x) await getData(x)
with open("./.temp/AirportDelays.i2m", 'a') as end: async with aiofiles.open("./.temp/AirportDelays.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/AirportDelays.i2m") dom = xml.dom.minidom.parse("./.temp/AirportDelays.i2m")
prettyXml = dom.toprettyxml(indent=" ") prettyXml = dom.toprettyxml(indent=" ")
with open("./.temp/AirportDelays.i2m", 'w') as g: async with aiofiles.open("./.temp/AirportDelays.i2m", 'w') as g:
g.write(prettyXml) await g.write(prettyXml)
g.close() await g.close()
files = [] files = []
commands = [] commands = []

66
recordGenerators/Alerts.py
View File

@ -9,6 +9,7 @@ import xml.dom.minidom
import shutil import shutil
import gzip import gzip
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio
import sys import sys
@ -26,15 +27,18 @@ headlineApiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
detailsApiKey = '21d8a80b3d6b444998a80b3d6b1449d3' detailsApiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
k = 0 k = 0
def getAlerts(location): async def getAlerts(location):
global k global k
fetchUrl = 'https://api.weather.com/v3/alerts/headlines?areaId=' + location + ':US&format=json&language=en-US&apiKey=' + headlineApiKey fetchUrl = 'https://api.weather.com/v3/alerts/headlines?areaId=' + location + ':US&format=json&language=en-US&apiKey=' + headlineApiKey
response = requests.get(fetchUrl) # response = requests.get(fetchUrl)
theCode = response.status_code # theCode = response.status_code
#Our global variables theCode = 0
async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
theCode = await r.status
#Set the actions based on response code #Set the actions based on response code
if theCode == 204: if theCode == 204:
@ -67,9 +71,9 @@ def getAlerts(location):
elif theCode == 200: elif theCode == 200:
pass pass
#Alright lets map our headline variables. # Map headline variables
l.debug('Found Alert for ' + location + '\n') l.debug('Found Alert for ' + location + '\n')
dataH = response.json() dataH = await r.json()
alertsRoot = dataH['alerts'] alertsRoot = dataH['alerts']
for x in alertsRoot: for x in alertsRoot:
@ -95,14 +99,15 @@ def getAlerts(location):
#theIdent = str(Identifier) #theIdent = str(Identifier)
try: try:
thecheck = open('./.temp/alertmanifest.txt', "r") async with aiofiles.open('./.temp/alertmanifest.txt', 'r' ) as checkFile:
check = thecheck.read() c = await checkFile.read()
if check.find(Identifier) != -1: if c.find(Identifier) != -1:
l.debug("Alert already sent...") l.debug(f"{Identifier} was sent already, skipping..")
return return
except FileNotFoundError: except FileNotFoundError:
l.warning("alert manifest does not exist (yet)") l.warning("alert manifest does not exist (yet)")
k += 1 #We have an alert to send! k += 1 #We have an alert to send!
#Lets Map Our Vocal Codes! #Lets Map Our Vocal Codes!
@ -307,40 +312,41 @@ def getAlerts(location):
alertMsg = '<BERecord id="0000" locationKey="' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '" isWxscan="0"><action>NOT_USED</action><BEHdr><bPIL>' + x['productIdentifier'] + '</bPIL><bWMOHdr>NOT_USED</bWMOHdr><bEvent><eActionCd eActionPriority="' + str(x['messageTypeCode']) + '">' + Action + '</eActionCd><eOfficeId eOfficeNm="' + x['officeName'] + '">' + x['officeCode'] + '</eOfficeId><ePhenom>' + x['phenomena'] + '</ePhenom><eSgnfcnc>' + x['significance'] + '</eSgnfcnc><eETN>' + x['eventTrackingNumber'] + '</eETN><eDesc>' + x['eventDescription'] + '</eDesc><eStTmUTC>NOT_USED</eStTmUTC><eEndTmUTC>' + EndTimeUTC + '</eEndTmUTC><eSvrty>' + str(x['severityCode']) + '</eSvrty><eTWCIId>NOT_USED</eTWCIId><eExpTmUTC>' + expireTimeUTC + '</eExpTmUTC></bEvent><bLocations><bLocCd bLoc="' + x['areaName'] + '" bLocTyp="' + locationType + '">' + location + '</bLocCd><bStCd bSt="' + x['adminDistrict'] + '">' + x['adminDistrictCode'] + '</bStCd><bUTCDiff>NOT_USED</bUTCDiff><bTzAbbrv>NOT_USED</bTzAbbrv><bCntryCd>NOT_USED</bCntryCd></bLocations><bSgmtChksum>' + x['identifier'] + '</bSgmtChksum><procTm>' + processTime + '</procTm></BEHdr><BEData><bIssueTmUTC>' + issueTimeUtc + '</bIssueTmUTC><bHdln><bHdlnTxt>' + x['headlineText'] + '</bHdlnTxt>' + vocalCode + '</bHdln><bParameter>NOT_USED</bParameter><bNarrTxt bNarrTxtLang="en-US"><bLn>' + description + '</bLn></bNarrTxt><bSrchRslt>NOT_USED</bSrchRslt></BEData><clientKey>' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '</clientKey></BERecord>' alertMsg = '<BERecord id="0000" locationKey="' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '" isWxscan="0"><action>NOT_USED</action><BEHdr><bPIL>' + x['productIdentifier'] + '</bPIL><bWMOHdr>NOT_USED</bWMOHdr><bEvent><eActionCd eActionPriority="' + str(x['messageTypeCode']) + '">' + Action + '</eActionCd><eOfficeId eOfficeNm="' + x['officeName'] + '">' + x['officeCode'] + '</eOfficeId><ePhenom>' + x['phenomena'] + '</ePhenom><eSgnfcnc>' + x['significance'] + '</eSgnfcnc><eETN>' + x['eventTrackingNumber'] + '</eETN><eDesc>' + x['eventDescription'] + '</eDesc><eStTmUTC>NOT_USED</eStTmUTC><eEndTmUTC>' + EndTimeUTC + '</eEndTmUTC><eSvrty>' + str(x['severityCode']) + '</eSvrty><eTWCIId>NOT_USED</eTWCIId><eExpTmUTC>' + expireTimeUTC + '</eExpTmUTC></bEvent><bLocations><bLocCd bLoc="' + x['areaName'] + '" bLocTyp="' + locationType + '">' + location + '</bLocCd><bStCd bSt="' + x['adminDistrict'] + '">' + x['adminDistrictCode'] + '</bStCd><bUTCDiff>NOT_USED</bUTCDiff><bTzAbbrv>NOT_USED</bTzAbbrv><bCntryCd>NOT_USED</bCntryCd></bLocations><bSgmtChksum>' + x['identifier'] + '</bSgmtChksum><procTm>' + processTime + '</procTm></BEHdr><BEData><bIssueTmUTC>' + issueTimeUtc + '</bIssueTmUTC><bHdln><bHdlnTxt>' + x['headlineText'] + '</bHdlnTxt>' + vocalCode + '</bHdln><bParameter>NOT_USED</bParameter><bNarrTxt bNarrTxtLang="en-US"><bLn>' + description + '</bLn></bNarrTxt><bSrchRslt>NOT_USED</bSrchRslt></BEData><clientKey>' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + x['eventTrackingNumber'] + '_' + x['officeCode'] + '</clientKey></BERecord>'
#Append BERecord #Append BERecord
with open('./.temp/BERecord.xml', "a") as b: async with aiofiles.open('./.temp/BERecord.xml', "a") as b:
b.write(alertMsg) await b.write(alertMsg)
b.close() await b.close()
#Add our alert to the manifest so we don't keep sending in the same alert every 60 seconds unless an update is issued. #Add our alert to the manifest so we don't keep sending in the same alert every 60 seconds unless an update is issued.
with open('./.temp/alertmanifest.txt', "a") as c: async with aiofiles.open('./.temp/alertmanifest.txt', "a") as c:
c.write('\n' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC'])) await c.write('\n' + location + '_' + x['phenomena'] + '_' + x['significance'] + '_' + str(x['processTimeUTC']))
c.close() await c.close()
# TODO: This should be converted into a function so it works better with async, that way we're not getting hung up on that time.sleep() call.
def makeRecord(): async def makeRecord():
loop = asyncio.get_running_loop()
global k global k
with open("./.temp/BERecord.xml", 'a') as BERecord:
BERecord.write('<Data type="BERecord">') # The BERecord XML doesn't need to be written if there's no alerts.
BERecord.close() if k > 0:
async with aiofiles.open("./.temp/BERecord.xml", 'a') as BERecord:
await BERecord.write('<Data type="BERecord">')
await BERecord.close()
for z in alertLocations: for z in alertLocations:
getAlerts(z) await getAlerts(z)
with open('./.temp/BERecord.xml', 'a') as BERecord: async with aiofiles.open('./.temp/BERecord.xml', 'a') as BERecord:
BERecord.write("</Data>") await BERecord.write("</Data>")
BERecord.close() await BERecord.close()
dom = xml.dom.minidom.parse("./.temp/BERecord.xml") dom = xml.dom.minidom.parse("./.temp/BERecord.xml")
pretty_xml_as_string = dom.toprettyxml(indent = " ") pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open("./.temp/BERecord.i2m", 'w') as h: async with aiofiles.open("./.temp/BERecord.i2m", 'w') as h:
h.write(pretty_xml_as_string[23:]) await h.write(pretty_xml_as_string[23:])
h.close() await h.close()
# If we don't need to send the i2 an alert, we don't need to gzip it.
if k > 0:
l.info("Sending alert(s) to the IntelliStar 2!") l.info("Sending alert(s) to the IntelliStar 2!")
with open("./.temp/BERecord.i2m", 'rb') as f_in: with open("./.temp/BERecord.i2m", 'rb') as f_in:
with gzip.open("./.temp/BERecord.gz", 'wb') as f_out: with gzip.open("./.temp/BERecord.gz", 'wb') as f_out:

36
recordGenerators/Breathing.py
View File

@ -6,6 +6,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio
sys.path.append("./py2lib") sys.path.append("./py2lib")
sys.path.append("./Util") sys.path.append("./Util")
@ -30,14 +31,14 @@ l.debug(coopIds, geocodes)
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(coopId, geocode): async def getData(coopId, geocode):
fetchUrl = f"https://api.weather.com/v2/indices/breathing/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v2/indices/breathing/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}"
data = ""
#Fetch data #Fetch data
async with aiohttp.ClientSession() as s:
response = requests.get(fetchUrl) async with s.get(fetchUrl) as r:
data = await r.text()
data = response.text
newData = data[63:-26] newData = data[63:-26]
@ -45,32 +46,33 @@ def getData(coopId, geocode):
#Write to .i2m file #Write to .i2m file
i2Doc = '<Breathing id="000000000" locationKey="' + str(coopId) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(coopId) + '</clientKey></Breathing>' i2Doc = '<Breathing id="000000000" locationKey="' + str(coopId) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(coopId) + '</clientKey></Breathing>'
f = open("./.temp/Breathing.i2m", "a") async with aiofiles.open("./.temp/Breathing.i2m", "a") as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeDataFile(): async def makeDataFile():
loop = asyncio.get_running_loop()
l.info("Writing a Breathing forecast record.") l.info("Writing a Breathing forecast record.")
header = '<Data type="Breathing">' header = '<Data type="Breathing">'
footer = '</Data>' footer = '</Data>'
with open("./.temp/Breathing.i2m", 'w') as doc: async with aiofiles.open("./.temp/Breathing.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for x, y in zip(coopIds, geocodes): for x, y in zip(coopIds, geocodes):
getData(x, y) await getData(x, y)
with open("./.temp/Breathing.i2m", 'a') as end: async with aiofiles.open("./.temp/Breathing.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/Breathing.i2m") dom = xml.dom.minidom.parse("./.temp/Breathing.i2m")
pretty_xml_as_string = dom.toprettyxml(indent = " ") pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open("./.temp/Breathing.i2m", "w") as g: async with aiofiles.open("./.temp/Breathing.i2m", "w") as g:
g.write(pretty_xml_as_string[23:]) await g.write(pretty_xml_as_string[23:])
g.close() await g.close()
files = [] files = []
commands = [] commands = []

46
recordGenerators/CurrentObservations.py
View File

@ -6,6 +6,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio
import sys import sys
sys.path.append("./py2lib") sys.path.append("./py2lib")
@ -34,50 +35,53 @@ for i in MPC.getMetroCities():
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(tecci, zipCode): async def getData(tecci, zipCode):
l.debug('Gathering data for location id ' + tecci)
fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/observations/current.xml?language=en-US&units=e&apiKey=' + apiKey fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/observations/current.xml?language=en-US&units=e&apiKey=' + apiKey
data = ""
#Fetch data async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
response = requests.get(fetchUrl) data = await r.text()
data = response.text
newData = data[67:-30] newData = data[67:-30]
l.debug('Gathering data for location id ' + tecci)
#Write to .i2m file #Write to .i2m file
i2Doc = '<CurrentObservations id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></CurrentObservations>' i2Doc = '<CurrentObservations id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></CurrentObservations>'
async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as f:
await f.write(i2Doc)
await f.close()
f = open("./.temp/CurrentObservations.i2m", "a") async def makeDataFile():
f.write(i2Doc) loop = asyncio.get_running_loop()
f.close()
def makeDataFile():
l.info("Writing a CurrentObservations record.") l.info("Writing a CurrentObservations record.")
header = '<Data type="CurrentObservations">' header = '<Data type="CurrentObservations">'
footer = '</Data>' footer = '</Data>'
with open("./.temp/CurrentObservations.i2m", 'w') as doc: async with aiofiles.open("./.temp/CurrentObservations.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for x, y in zip(tecciId, zipCodes): for x, y in zip(tecciId, zipCodes):
getData(x, y) await getData(x, y)
with open("./.temp/CurrentObservations.i2m", 'a') as end: async with aiofiles.open("./.temp/CurrentObservations.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/CurrentObservations.i2m") dom = xml.dom.minidom.parse("./.temp/CurrentObservations.i2m")
pretty_xml_as_string = dom.toprettyxml(indent = " ") pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open("./.temp/CurrentObservations.i2m", "w") as g: async with aiofiles.open("./.temp/CurrentObservations.i2m", "w") as g:
g.write(pretty_xml_as_string[23:]) await g.write(pretty_xml_as_string[23:])
g.close() await g.close()
files = [] files = []
commands = [] commands = []
"""
TODO: This can be ran in a seperate thread using loop.run_in_executor() according to the python discord.
! This should probably be implemented ASAP.
"""
with open("./.temp/CurrentObservations.i2m", 'rb') as f_in: with open("./.temp/CurrentObservations.i2m", 'rb') as f_in:
with gzip.open("./.temp/CurrentObservations.gz", 'wb') as f_out: with gzip.open("./.temp/CurrentObservations.gz", 'wb') as f_out:
shutil.copyfileobj(f_in, f_out) shutil.copyfileobj(f_in, f_out)
@ -88,7 +92,7 @@ def makeDataFile():
command = commands.append('<MSG><Exec workRequest="storeData(File={0},QGROUP=__CurrentObservations__,Feed=CurrentObservations)" /><GzipCompressedMsg fname="CurrentObservations" /></MSG>') command = commands.append('<MSG><Exec workRequest="storeData(File={0},QGROUP=__CurrentObservations__,Feed=CurrentObservations)" /><GzipCompressedMsg fname="CurrentObservations" /></MSG>')
numFiles = len(files) numFiles = len(files)
bit.sendFile(files, commands, numFiles, 0) await loop.run_in_executor(bit.sendFile(files, commands, numFiles, 0))
os.remove("./.temp/CurrentObservations.i2m") os.remove("./.temp/CurrentObservations.i2m")
os.remove("./.temp/CurrentObservations.gz") os.remove("./.temp/CurrentObservations.gz")

37
recordGenerators/DailyForecast.py
View File

@ -6,6 +6,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio
sys.path.append("./py2lib") sys.path.append("./py2lib")
sys.path.append("./Util") sys.path.append("./Util")
@ -32,14 +33,13 @@ for i in MPC.getMetroCities():
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(tecci, zipCode): async def getData(tecci, zipCode):
fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/daily/7day.xml?language=en-US&units=e&apiKey=' + apiKey fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/daily/7day.xml?language=en-US&units=e&apiKey=' + apiKey
data = ""
#Fetch data async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
response = requests.get(fetchUrl) data = await r.text()
data = response.text
newData = data[61:-24] newData = data[61:-24]
@ -47,32 +47,33 @@ def getData(tecci, zipCode):
#Write to .i2m file #Write to .i2m file
i2Doc = '<DailyForecast id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></DailyForecast>' i2Doc = '<DailyForecast id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></DailyForecast>'
f = open("./.temp/DailyForecast.i2m", "a") async with aiofiles.open('./.temp/DailyForecast.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeDataFile(): async def makeDataFile():
loop = asyncio.get_running_loop()
l.info("Writing a DailyForecast record.") l.info("Writing a DailyForecast record.")
header = '<Data type="DailyForecast">' header = '<Data type="DailyForecast">'
footer = '</Data>' footer = '</Data>'
with open("./.temp/DailyForecast.i2m", 'w') as doc: async with aiofiles.open("./.temp/DailyForecast.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for x, y in zip(tecciId, zipCodes): for x, y in zip(tecciId, zipCodes):
getData(x, y) await getData(x, y)
with open("./.temp/DailyForecast.i2m", 'a') as end: async with aiofiles.open("./.temp/DailyForecast.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/DailyForecast.i2m") dom = xml.dom.minidom.parse("./.temp/DailyForecast.i2m")
pretty_xml_as_string = dom.toprettyxml(indent = " ") pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open("./.temp/DailyForecast.i2m", "w") as g: async with aiofiles.open("./.temp/DailyForecast.i2m", "w") as g:
g.write(pretty_xml_as_string[23:]) await g.write(pretty_xml_as_string[23:])
g.close() await g.close()
files = [] files = []
commands = [] commands = []

42
recordGenerators/HeatingAndCooling.py
View File

@ -7,6 +7,7 @@ import records.LFRecord as LFR
import gzip import gzip
from os import remove from os import remove
import xml.dom.minidom import xml.dom.minidom
import aiohttp, aiofiles, asyncio
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install() coloredlogs.install()
@ -20,46 +21,49 @@ for i in MPC.getPrimaryLocations():
apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" apiKey = "21d8a80b3d6b444998a80b3d6b1449d3"
def getData(coopId, geocode): async def getData(coopId, geocode):
fetchUrl = f"https://api.weather.com/v2/indices/heatCool/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v2/indices/heatCool/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}"
data = ""
res = requests.get(fetchUrl) async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
if res.status_code != 200: if r.status != 200:
l.error("DO NOT REPORT THE ERROR BELOW") l.error(f"Failed to write HeatingAndCooling record -- Status code {r.status}")
l.error(f"Failed to write HeatingAndCooling record -- Status code {res.status_code}")
return return
data = res.text data = await r.text()
# data = res.text
newData = data[63:-26] newData = data[63:-26]
i2Doc = f'\n <HeatingAndCooling id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </HeatingAndCooling>' i2Doc = f'\n <HeatingAndCooling id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </HeatingAndCooling>'
f = open('./.temp/HeatingAndCooling.i2m', 'a') async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeRecord(): async def makeRecord():
loop = asyncio.get_running_loop()
l.info("Writing HeatingAndCooling record.") l.info("Writing HeatingAndCooling record.")
header = '<Data type="HeatingAndCooling">' header = '<Data type="HeatingAndCooling">'
footer = '</Data>' footer = '</Data>'
with open('./.temp/HeatingAndCooling.i2m', 'a') as doc: async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as doc:
doc.write(header) await doc.write(header)
for (x, y) in zip(coopIds, geocodes): for (x, y) in zip(coopIds, geocodes):
getData(x,y) await getData(x,y)
with open('./.temp/HeatingAndCooling.i2m', 'a') as end: async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse('./.temp/HeatingAndCooling.i2m') dom = xml.dom.minidom.parse('./.temp/HeatingAndCooling.i2m')
xmlPretty = dom.toprettyxml(indent= " ") xmlPretty = dom.toprettyxml(indent= " ")
with open('./.temp/HeatingAndCooling.i2m', 'w') as g: async with aiofiles.open('./.temp/HeatingAndCooling.i2m', 'w') as g:
g.write(xmlPretty[23:]) await g.write(xmlPretty[23:])
g.close() await g.close()
# Compresss i2m to gzip # Compresss i2m to gzip

40
recordGenerators/HourlyForecast.py
View File

@ -5,6 +5,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging,coloredlogs import logging,coloredlogs
import aiohttp, aiofiles, asyncio, asyncio
import sys import sys
sys.path.append("./py2lib") sys.path.append("./py2lib")
@ -32,46 +33,49 @@ for i in MPC.getMetroCities():
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(tecci, zipCode): async def getData(tecci, zipCode):
l.debug('Gathering data for location id ' + tecci)
fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/hourly/360hour.xml?language=en-US&units=e&apiKey=' + apiKey fetchUrl = 'https://api.weather.com/v1/location/' + zipCode + ':4:US/forecast/hourly/360hour.xml?language=en-US&units=e&apiKey=' + apiKey
data = ""
#Fetch data #Fetch data
async with aiohttp.ClientSession() as s:
response = requests.get(fetchUrl) async with s.get(fetchUrl) as r:
data = await r.text()
data = response.text
newData = data[48:-11] newData = data[48:-11]
l.debug('Gathering data for location id ' + tecci)
#Write to .i2m file #Write to .i2m file
i2Doc = '<HourlyForecast id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></HourlyForecast>' i2Doc = '<HourlyForecast id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></HourlyForecast>'
f = open("./.temp/HourlyForecast.i2m", "a") async with aiofiles.open('./.temp/HourlyForecast.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeDataFile():
async def makeDataFile():
loop = asyncio.get_running_loop()
l.info("Writing an HourlyForecast record.") l.info("Writing an HourlyForecast record.")
header = '<Data type="HourlyForecast">' header = '<Data type="HourlyForecast">'
footer = '</Data>' footer = '</Data>'
with open("./.temp/HourlyForecast.i2m", 'w') as doc: async with aiofiles.open("./.temp/HourlyForecast.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for x, y in zip(tecciId, zipCodes): for x, y in zip(tecciId, zipCodes):
getData(x, y) await getData(x, y)
with open("./.temp/HourlyForecast.i2m", 'a') as end: async with aiofiles.open("./.temp/HourlyForecast.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/HourlyForecast.i2m") dom = xml.dom.minidom.parse("./.temp/HourlyForecast.i2m")
pretty_xml_as_string = dom.toprettyxml(indent = " ") pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open("./.temp/HourlyForecast.i2m", "w") as g: async with aiofiles.open("./.temp/HourlyForecast.i2m", "w") as g:
g.write(pretty_xml_as_string[23:]) await g.write(pretty_xml_as_string[23:])
g.close() await g.close()
files = [] files = []
commands = [] commands = []

42
recordGenerators/MosquitoActivity.py
View File

@ -7,6 +7,7 @@ import records.LFRecord as LFR
import gzip import gzip
from os import remove from os import remove
import xml.dom.minidom import xml.dom.minidom
import aiohttp, aiofiles, asyncio
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install() coloredlogs.install()
@ -20,46 +21,49 @@ for i in MPC.getPrimaryLocations():
apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" apiKey = "21d8a80b3d6b444998a80b3d6b1449d3"
def getData(coopId, geocode): async def getData(coopId, geocode):
fetchUrl = f"https://api.weather.com/v2/indices/mosquito/daily/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v2/indices/mosquito/daily/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}"
data = ""
res = requests.get(fetchUrl) async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
if res.status_code != 200: if r.status != 200:
l.error("DO NOT REPORT THE ERROR BELOW") l.error(f"Failed to write MosquitoActivity record -- status code {r.status}")
l.error(f"Failed to write MosquitoActivity record -- Status code {res.status_code}")
return return
data = res.text data = await r.text()
newData = data[63:-26] newData = data[63:-26]
i2Doc = f'\n <MosquitoActivity id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </MosquitoActivity>' i2Doc = f'\n <MosquitoActivity id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </MosquitoActivity>'
f = open('./.temp/MosquitoActivity.i2m', 'a') async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeRecord(): async def makeRecord():
loop = asyncio.get_running_loop()
l.info("Writing MosquitoActivity record.") l.info("Writing MosquitoActivity record.")
header = '<Data type="MosquitoActivity">' header = '<Data type="MosquitoActivity">'
footer = '</Data>' footer = '</Data>'
with open('./.temp/MosquitoActivity.i2m', 'a') as doc: async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as doc:
doc.write(header) await doc.write(header)
for (x, y) in zip(coopIds, geocodes): for (x, y) in zip(coopIds, geocodes):
getData(x,y) await getData(x,y)
with open('./.temp/MosquitoActivity.i2m', 'a') as end: async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse('./.temp/MosquitoActivity.i2m') dom = xml.dom.minidom.parse('./.temp/MosquitoActivity.i2m')
xmlPretty = dom.toprettyxml(indent= " ") xmlPretty = dom.toprettyxml(indent= " ")
with open('./.temp/MosquitoActivity.i2m', 'w') as g: async with aiofiles.open('./.temp/MosquitoActivity.i2m', 'w') as g:
g.write(xmlPretty[23:]) await g.write(xmlPretty[23:])
g.close() await g.close()
# Compresss i2m to gzip # Compresss i2m to gzip

37
recordGenerators/PollenForecast.py
View File

@ -6,6 +6,7 @@ import os
import shutil import shutil
import xml.dom.minidom import xml.dom.minidom
import logging, coloredlogs import logging, coloredlogs
import aiohttp, aiofiles, asyncio
sys.path.append("./py2lib") sys.path.append("./py2lib")
sys.path.append("./Util") sys.path.append("./Util")
@ -31,14 +32,13 @@ l.debug(pollenIds, geocodes)
apiKey = '21d8a80b3d6b444998a80b3d6b1449d3' apiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
def getData(pollenId, geocode): async def getData(pollenId, geocode):
fetchUrl = f"https://api.weather.com/v2/indices/pollen/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v2/indices/pollen/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}"
data = ""
#Fetch data #Fetch data
async with aiohttp.ClientSession() as s:
response = requests.get(fetchUrl) async with s.get(fetchUrl) as r:
data = await r.text()
data = response.text
newData = data[63:-26] newData = data[63:-26]
@ -46,32 +46,33 @@ def getData(pollenId, geocode):
#Write to .i2m file #Write to .i2m file
i2Doc = '<PollenForecast id="000000000" locationKey="' + str(pollenId) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(pollenId) + '</clientKey></PollenForecast>' i2Doc = '<PollenForecast id="000000000" locationKey="' + str(pollenId) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(pollenId) + '</clientKey></PollenForecast>'
f = open("./.temp/PollenForecast.i2m", "a") async with aiofiles.open("./.temp/PollenForecast.i2m", "a") as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeDataFile(): async def makeDataFile():
loop = asyncio.get_running_loop()
l.info("Writing a PollenForecast record.") l.info("Writing a PollenForecast record.")
header = '<Data type="PollenForecast">' header = '<Data type="PollenForecast">'
footer = '</Data>' footer = '</Data>'
with open("./.temp/PollenForecast.i2m", 'w') as doc: async with aiofiles.open("./.temp/PollenForecast.i2m", 'w') as doc:
doc.write(header) await doc.write(header)
for x, y in zip(pollenIds, geocodes): for x, y in zip(pollenIds, geocodes):
getData(x, y) await getData(x, y)
with open("./.temp/PollenForecast.i2m", 'a') as end: async with aiofiles.open("./.temp/PollenForecast.i2m", 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse("./.temp/PollenForecast.i2m") dom = xml.dom.minidom.parse("./.temp/PollenForecast.i2m")
pretty_xml_as_string = dom.toprettyxml(indent = " ") pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open("./.temp/PollenForecast.i2m", "w") as g: async with aiofiles.open("./.temp/PollenForecast.i2m", "w") as g:
g.write(pretty_xml_as_string[23:]) await g.write(pretty_xml_as_string[23:])
g.close() await g.close()
files = [] files = []
commands = [] commands = []

44
recordGenerators/TideForecast.py
View File

@ -1,6 +1,4 @@
import shutil import shutil
from xmlrpc.client import DateTime
import requests
import logging,coloredlogs import logging,coloredlogs
import datetime import datetime
from py2Lib import bit from py2Lib import bit
@ -9,6 +7,7 @@ import records.LFRecord as LFR
import gzip import gzip
from os import remove from os import remove
import xml.dom.minidom import xml.dom.minidom
import aiohttp, aiofiles, asyncio
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install() coloredlogs.install()
@ -22,31 +21,34 @@ for i in MPC.getTideStations():
apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" apiKey = "21d8a80b3d6b444998a80b3d6b1449d3"
def getData(tideStation, geocode): async def getData(tideStation, geocode):
today = datetime.date.today() today = datetime.date.today()
startDate = today.strftime('%Y%m%d') startDate = today.strftime('%Y%m%d')
endDate_unformatted = datetime.datetime.strptime(startDate, '%Y%m%d') + datetime.timedelta(days=5) endDate_unformatted = datetime.datetime.strptime(startDate, '%Y%m%d') + datetime.timedelta(days=5)
endDate = endDate_unformatted.strftime('%Y%m%d') endDate = endDate_unformatted.strftime('%Y%m%d')
data = ""
fetchUrl = f"https://api.weather.com/v1/geocode/{geocode}/forecast/tides.xml?language=en-US&units=e&startDate={startDate}&endDate={endDate}&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v1/geocode/{geocode}/forecast/tides.xml?language=en-US&units=e&startDate={startDate}&endDate={endDate}&apiKey={apiKey}"
res = requests.get(fetchUrl) async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
if res.status_code != 200: if r.status != 200:
l.error("DO NOT REPORT THE ERROR BELOW") l.error(f"Failed to write TideForecast -- status code {r.status}")
l.error(f"Failed to write TidesForecast record -- Status code {res.status_code}")
return return
data = res.text data = await r.text()
newData = data[53:-16] newData = data[53:-16]
i2Doc = f'\n <TidesForecast id="000000000" locationKey="{tideStation}" isWxScan="0">\n {newData}\n <clientKey>{tideStation}</clientKey>\n </TidesForecast>' i2Doc = f'\n <TidesForecast id="000000000" locationKey="{tideStation}" isWxScan="0">\n {newData}\n <clientKey>{tideStation}</clientKey>\n </TidesForecast>'
f = open('./.temp/TidesForecast.i2m', 'a') async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeRecord(): async def makeRecord():
loop = asyncio.get_running_loop()
if len(tideStations) < 1: if len(tideStations) < 1:
l.debug("Skipping TidesForecast -- No locations.") l.debug("Skipping TidesForecast -- No locations.")
return return
@ -56,21 +58,21 @@ def makeRecord():
header = '<Data type="TidesForecast">' header = '<Data type="TidesForecast">'
footer = '</Data>' footer = '</Data>'
with open('./.temp/TidesForecast.i2m', 'a') as doc: async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as doc:
doc.write(header) await doc.write(header)
for (x, y) in zip(tideStations, geocodes): for (x, y) in zip(tideStations, geocodes):
getData(x,y) await getData(x,y)
with open('./.temp/TidesForecast.i2m', 'a') as end: async with aiofiles.open('./.temp/TidesForecast.i2m', 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse('./.temp/TidesForecast.i2m') dom = xml.dom.minidom.parse('./.temp/TidesForecast.i2m')
xmlPretty = dom.toprettyxml(indent= " ") xmlPretty = dom.toprettyxml(indent= " ")
with open('./.temp/TidesForecast.i2m', 'w') as g: async with aiofiles.open('./.temp/TidesForecast.i2m', 'w') as g:
g.write(xmlPretty[23:]) await g.write(xmlPretty[23:])
g.close() await g.close()
# Compresss i2m to gzip # Compresss i2m to gzip

41
recordGenerators/WateringNeeds.py
View File

@ -7,6 +7,7 @@ import records.LFRecord as LFR
import gzip import gzip
from os import remove from os import remove
import xml.dom.minidom import xml.dom.minidom
import aiohttp, aiofiles, asyncio
l = logging.getLogger(__name__) l = logging.getLogger(__name__)
coloredlogs.install() coloredlogs.install()
@ -20,46 +21,48 @@ for i in MPC.getPrimaryLocations():
apiKey = "21d8a80b3d6b444998a80b3d6b1449d3" apiKey = "21d8a80b3d6b444998a80b3d6b1449d3"
def getData(coopId, geocode): async def getData(coopId, geocode):
fetchUrl = f"https://api.weather.com/v2/indices/wateringNeeds/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}" fetchUrl = f"https://api.weather.com/v2/indices/wateringNeeds/daypart/7day?geocode={geocode}&language=en-US&format=xml&apiKey={apiKey}"
data = ""
res = requests.get(fetchUrl) async with aiohttp.ClientSession() as s:
async with s.get(fetchUrl) as r:
if res.status_code != 200: if r.status != 200:
l.error("DO NOT REPORT THE ERROR BELOW") l.error(f"Failed to WateringNeeds -- status code {r.status}")
l.error(f"Failed to write WateringNeeds record -- Status code {res.status_code}")
return return
data = res.text data = await r.text()
newData = data[63:-26] newData = data[63:-26]
i2Doc = f'\n <WateringNeeds id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </WateringNeeds>' i2Doc = f'\n <WateringNeeds id="000000000" locationKey="{coopId}" isWxScan="0">\n {newData}\n <clientKey>{coopId}</clientKey>\n </WateringNeeds>'
f = open('./.temp/WateringNeeds.i2m', 'a') async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as f:
f.write(i2Doc) await f.write(i2Doc)
f.close() await f.close()
def makeRecord(): async def makeRecord():
loop = asyncio.get_running_loop()
l.info("Writing WateringNeeds record.") l.info("Writing WateringNeeds record.")
header = '<Data type="WateringNeeds">' header = '<Data type="WateringNeeds">'
footer = '</Data>' footer = '</Data>'
with open('./.temp/WateringNeeds.i2m', 'a') as doc: async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as doc:
doc.write(header) await doc.write(header)
for (x, y) in zip(coopIds, geocodes): for (x, y) in zip(coopIds, geocodes):
getData(x,y) await getData(x,y)
with open('./.temp/WateringNeeds.i2m', 'a') as end: async with aiofiles.open('./.temp/WateringNeeds.i2m', 'a') as end:
end.write(footer) await end.write(footer)
dom = xml.dom.minidom.parse('./.temp/WateringNeeds.i2m') dom = xml.dom.minidom.parse('./.temp/WateringNeeds.i2m')
xmlPretty = dom.toprettyxml(indent= " ") xmlPretty = dom.toprettyxml(indent= " ")
with open('./.temp/WateringNeeds.i2m', 'w') as g: async with aiofiles.open('./.temp/WateringNeeds.i2m', 'w') as g:
g.write(xmlPretty[23:]) await g.write(xmlPretty[23:])
g.close() await g.close()
# Compresss i2m to gzip # Compresss i2m to gzip

0
recordGenerators/__init__.py Normal file
View File

4
records/LFRecord.py
View File

@ -38,7 +38,3 @@ def getLatLong(locId: str):
def getLocationInfo(locId: str): def getLocationInfo(locId: str):
pass pass
print(getCoopId('USAZ0278'))
print(getZip('USAZ0278'))
print(getLatLong('USAZ0278'))