diff --git a/main.py b/main.py index bc268c4..7f32d36 100644 --- a/main.py +++ b/main.py @@ -1,19 +1,25 @@ import asyncio -# from re import A +import logging,coloredlogs from recordGenerators import DailyForecast,CurrentObservations,HourlyForecast,AirQuality,AirportDelays,PollenForecast,Breathing + +l = logging.getLogger(__name__) +coloredlogs.install(logger=l) + + """ CurrentConditions: Every 5 minutes Daily Forecasts, Hourlies, etc: 60 minutes Alerts: 5 minutes """ +l.info("Starting i2RecordCollector") +l.info("Developed by mewtek32, Floppaa, and Goldblaze") -print("i2MessageEncoder-Python\nDeveloped by mewtek\nData record generators by Floppaa & Goldblaze") async def FiveMinUpdaters(): while True: CurrentObservations.makeDataFile() - print("Sleeping for 5 minutes...") + l.debug("Sleeping for 5 minutes...") await asyncio.sleep(300) @@ -25,7 +31,7 @@ async def HourUpdaters(): PollenForecast.makeDataFile() AirportDelays.writeData() Breathing.makeDataFile() - print("Sleeping for an hour...") + l.debug("Sleeping for an hour...") await asyncio.sleep(3600) loop = asyncio.get_event_loop() diff --git a/py2Lib/bit.py b/py2Lib/bit.py index e4f0f90..c21a29f 100644 --- a/py2Lib/bit.py +++ b/py2Lib/bit.py @@ -5,6 +5,10 @@ import struct import binascii import math import time +import logging,coloredlogs + +l = logging.getLogger(__name__) +coloredlogs.install() MCAST_GRP = '224.1.1.77' MCAST_IF = '127.0.0.1' @@ -23,7 +27,7 @@ def sendFile(files, commands, numSgmts, Pri): elif Pri == 1: MCAST_PORT = 7788 else: - print("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...") + l.critical("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...") exit() #Get the next message ID with open('C:\\Clips\\msgId.txt', "r") as f: @@ -37,9 +41,9 @@ def sendFile(files, commands, numSgmts, Pri): h.close() segnmNum = 0 if Pri == 0: - print("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") + l.info("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") elif Pri == 1: - print("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") + l.info("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") startFlag = False for x, y in zip(files, commands): @@ -85,7 +89,7 @@ def sendFile(files, commands, numSgmts, Pri): conn.sendto(packetHeader + fec + data + theNull, (MCAST_GRP, MCAST_PORT)) else: conn.sendto(packetHeader + fec + data, (MCAST_GRP, MCAST_PORT)) - print(packet_count) + l.debug(packet_count) packet_count += 1 j += 1 @@ -115,7 +119,7 @@ def sendCommand(command, Pri, msgNum = None): elif Pri == 1: MCAST_PORT = 7788 else: - print("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...") + l.critical("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...") exit() #Get the next message ID with open('C:\\Clips\\msgId.txt', "r") as f: @@ -129,9 +133,9 @@ def sendCommand(command, Pri, msgNum = None): h.close() segnmNum = 0 if Pri == 0: - print("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") + l.info("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") elif Pri == 1: - print("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") + l.info("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....") startFlag = False for x in command: @@ -181,7 +185,7 @@ def sendCommand(command, Pri, msgNum = None): conn.sendto(packetHeader + fec + data + theNull, (MCAST_GRP, MCAST_PORT)) else: conn.sendto(packetHeader + fec + data, (MCAST_GRP, MCAST_PORT)) - print(packet_count) + l.debug(packet_count) packet_count += 1 j += 1 diff --git a/radar/TWCRadarProcessor.py b/radar/TWCRadarProcessor.py index cbde682..4045608 100644 --- a/radar/TWCRadarProcessor.py +++ b/radar/TWCRadarProcessor.py @@ -7,6 +7,7 @@ import aiohttp import json import time as epochTime import requests +import logging,coloredlogs from RadarProcessor import * from os import path, mkdir, listdir, remove, cpu_count @@ -18,6 +19,9 @@ from wand.color import Color radarType = "Radar-US" +l = logging.getLogger(__name__) +coloredlogs.install() + upperLeftX,upperLeftY,lowerRightX,lowerRightY = 0,0,0,0 xStart,xEnd,yStart,yEnd = 0,0,0,0 imgW = 0 @@ -29,7 +33,7 @@ import bit async def getValidTimestamps(boundaries:ImageBoundaries) -> list: """Gets all valid UNIX timestamps for the TWCRadarMosaic product """ - print("Getting timestamps for the radar..") + l.info("Getting timestamps for the radar..") times = [] async with aiohttp.ClientSession() as session: @@ -44,12 +48,12 @@ async def getValidTimestamps(boundaries:ImageBoundaries) -> list: # Don't add frames that aren't at the correct interval if (time % boundaries.ImageInterval != 0): - print(f"Ignoring {time} -- Not at the correct frame interval.") + l.debug(f"Ignoring {time} -- Not at the correct frame interval.") continue # Don't add frames that are expired if (time < (datetime.utcnow().timestamp() - epochTime.time()) / 1000 - boundaries.Expiration): - print(f"Ignoring {time} -- Expired.") + l.debug(f"Ignoring {time} -- Expired.") continue times.append(time) @@ -63,13 +67,13 @@ def downloadRadarTile(url, p, fn): # Make the path if it doesn't exist if exists(f"tiles/output/{ts}.tiff"): - print("Not downloading tiles for timestamp " + str(ts) + " since a frame for it already exists." ) + l.debug("Not downloading tiles for timestamp " + str(ts) + " since a frame for it already exists." ) download = False if not path.exists(p): mkdir(p) - print(f"Download {ts}") + l.debug(f"Download {ts}") if exists(f"{p}/{fn}"): - print(f"Not downloading new tiles for {ts} as they already exist.") + l.debug(f"Not downloading new tiles for {ts} as they already exist.") download = False if (img.status_code == 200 and download): @@ -77,7 +81,7 @@ def downloadRadarTile(url, p, fn): for data in img: tile.write(data) elif (img.status_code != 200): - print("ERROR DOWNLOADING " + p + "\nSTATUS CODE " + str(img.status_code)) + l.error("ERROR DOWNLOADING " + p + "\nSTATUS CODE " + str(img.status_code)) elif (download == False): pass @@ -206,6 +210,7 @@ def getTime(timestamp) -> str: async def makeRadarImages(): """ Creates proper radar frames for the i2 """ + l.info("Downloading frames for the Regional Radar...") combinedCoordinates = [] @@ -221,7 +226,7 @@ async def makeRadarImages(): # Get rid of invalid radar frames for i in listdir('tiles/output'): if i.split('.')[0] not in [str(x) for x in times] and i != "Thumbs.db": - print(f"Deleting {i} as it is no longer valid.") + l.debug(f"Deleting {i} as it is no longer valid.") remove("tiles/output/" + i) # Collect coordinates for the frame tiles @@ -242,7 +247,7 @@ async def makeRadarImages(): paths.append(f"tiles/{times[i]}") filenames.append(f"{times[i]}_{combinedCoordinates[c].x}_{combinedCoordinates[c].y}.png") - print(len(urls)) + l.debug(len(urls)) if len(urls) != 0 and len(urls) >= 6: with Pool(cpu_count() - 1) as p: p.starmap(downloadRadarTile, zip(urls, paths, filenames)) @@ -254,7 +259,7 @@ async def makeRadarImages(): p.close() p.join() elif len(urls) == 0: - print("No new radar frames need to be downloaded.") + l.info("No new radar frames need to be downloaded.") return # Stitch them all together! @@ -270,7 +275,7 @@ async def makeRadarImages(): # Stitch the frames together for i in range(0, len(imgsToGenerate)): if not exists(F"tiles/output/{times[i]}.tiff"): - print(f"Generate frame for {times[i]}") + l.debug(f"Generate frame for {times[i]}") for c in combinedCoordinates: path = f"tiles/{times[i]}/{times[i]}_{c.x}_{c.y}.png" @@ -290,7 +295,7 @@ async def makeRadarImages(): # Composite images for the i2 for img in framesToComposite: - print("Attempting to composite " + img) + l.debug("Attempting to composite " + img) # Crop the radar images something that the i2 will actually take img_raw = wandImage(filename=img) @@ -315,6 +320,8 @@ async def makeRadarImages(): bit.sendFile([finished[i]], [commands[i]], 1, 0) + l.info("Downloaded and sent Regional Radar frames!") + # print(getTime(1665880800)) diff --git a/radar/TWCSatRadProcessor.py b/radar/TWCSatRadProcessor.py new file mode 100644 index 0000000..a2ddb37 --- /dev/null +++ b/radar/TWCSatRadProcessor.py @@ -0,0 +1,69 @@ +import asyncio +import aiohttp +import time as epochTime +import datetime +import requests +from RadarProcessor import * +from os import mkdir, path +from genericpath import exists + + +upperLeftX,upperLeftY,lowerRightX,lowerRightY = 0,0,0,0 +xStart,xEnd,yStart,yEnd = 0,0,0,0 +imgW = 0 +imgH = 0 + +async def getValidTimestamps(boundaries:ImageBoundaries) -> list: + """Gets all valid UNIX timestamps for the TWCRadarMosaic product """ + print("Getting timestamps for the radar..") + times = [] + + async with aiohttp.ClientSession() as session: + url = "https://api.weather.com/v3/TileServer/series/productSet?apiKey=21d8a80b3d6b444998a80b3d6b1449d3&filter=twcRadarMosaic" + async with session.get(url) as r: + response = await r.json() + + for t in range(0, len(response['seriesInfo']['twcRadarMosaic']['series'])): + + if (t <= 35): + time = response['seriesInfo']['twcRadarMosaic']['series'][t]['ts'] + + # Don't add frames that aren't at the correct interval + if (time % boundaries.ImageInterval != 0): + print(f"Ignoring {time} -- Not at the correct frame interval.") + continue + + # Don't add frames that are expired + if (time < (datetime.utcnow().timestamp() - epochTime.time()) / 1000 - boundaries.Expiration): + print(f"Ignoring {time} -- Expired.") + continue + + times.append(time) + + return times + +def downloadRadarTile(url, p, fn): + img = requests.get(url, stream=True) + ts = fn.split("_")[0] + download = True + + # Make the path if it doesn't exist + if exists(f"tiles/output/{ts}.tiff"): + print("Not downloading tiles for timestamp " + str(ts) + " since a frame for it already exists." ) + download = False + if not path.exists(p): + mkdir(p) + print(f"Download {ts}") + if exists(f"{p}/{fn}"): + print(f"Not downloading new tiles for {ts} as they already exist.") + download = False + + if (img.status_code == 200 and download): + with open(f'{p}/{fn}', 'wb') as tile: + for data in img: + tile.write(data) + elif (img.status_code != 200): + print("ERROR DOWNLOADING " + p + "\nSTATUS CODE " + str(img.status_code)) + elif (download == False): + pass + diff --git a/recordGenerators/AirQuality.py b/recordGenerators/AirQuality.py index 71bfe97..29b6f30 100644 --- a/recordGenerators/AirQuality.py +++ b/recordGenerators/AirQuality.py @@ -3,6 +3,10 @@ import gzip import os import shutil import xml.dom.minidom +import logging,coloredlogs + +l = logging.getLogger(__name__) +coloredlogs.install() import sys sys.path.append("./py2lib") @@ -44,9 +48,9 @@ def writeData(): for i in epaIds: if i == None: - print(f"No EPA ID found for location -- Skipping.") + l.debug(f"No EPA ID found for location -- Skipping.") else: - print(f"EPA ID found for location! Writing data for Air Quality.") + l.debug(f"EPA ID found for location! Writing data for Air Quality.") workingEpaIds.append(i) useData = True @@ -54,6 +58,7 @@ def writeData(): # Check to see if we even have EPA ids, as some areas don't have air quality reports if (useData): try: + l.info("Writing an AirQuality record.") header = '' footer = "" @@ -90,9 +95,10 @@ def writeData(): os.remove("D:\\AirQuality.i2m") os.remove("D:\\AirQuality.gz") except Exception as e: - print("AirQuality failed to write, problably was expired and was sticking around in the IBM api.") + l.error("DO NOT REPORT THE ERROR BELOW") + l.error("Failed to write an AirQuality record.") else: - print("Ignoring AirQuality data collection -- No working EPA Ids.") + l.info("Not writing an AirQuality record due to a lack of working EPA ids.") diff --git a/recordGenerators/AirportDelays.py b/recordGenerators/AirportDelays.py index 6e625ef..e248cc6 100644 --- a/recordGenerators/AirportDelays.py +++ b/recordGenerators/AirportDelays.py @@ -3,6 +3,7 @@ import gzip import os import shutil import xml.dom.minidom +import logging,coloredlogs import sys sys.path.append("./py2lib") @@ -12,6 +13,9 @@ import bit import MachineProductCfg as MPC import LFRecord as LFR +l = logging.getLogger(__name__) +coloredlogs.install() + locationIds = [] zipCodes = [] airports = [] @@ -35,7 +39,6 @@ def getData(airport): # Write to i2doc file i2Doc = f'' + '' + newData + f'{airport}' - print(f"[AIRPORT DELAYS] Writing airport delay data for {airport}") f = open("D:\\AirportDelays.i2m", 'a') f.write(i2Doc) @@ -50,13 +53,14 @@ def writeData(): res = requests.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}") if (res.status_code != 200): - print(f"[AIRPORT DELAYS] No delays for {x} found, skipping..") + l.debug(f"[AIRPORT DELAYS] No delays for {x} found, skipping..") else: airportsWithDelays.append(x) - print(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..") + l.debug(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..") useData = True if (useData): + l.info("Writing an AirportDelays record.") header = '' footer = "" @@ -93,4 +97,4 @@ def writeData(): os.remove("D:\\AirportDelays.i2m") os.remove("D:\\AirportDelays.gz") else: - print("[AIRPORT DELAYS] Not writing AirportDelays -- Either no delays found, or the API is broken.") + l.info("No airport delays found.") diff --git a/recordGenerators/Alerts.py b/recordGenerators/Alerts.py index ba9b7d1..6544e2c 100644 --- a/recordGenerators/Alerts.py +++ b/recordGenerators/Alerts.py @@ -2,6 +2,7 @@ import requests import json import os from datetime import datetime,timedelta +from Util.MachineProductCfg import getZones import time import pytz import xml.dom.minidom @@ -10,7 +11,7 @@ import gzip import py2Lib.bit as bit #Zones/Counties to fetch alerts for -interestList = ['FLZ151', 'FLC057', 'FLZ149', 'FLZ249', 'FLC101'] # TODO: Grab these automatically from MachineProductCfg.xml +zones = getZones() #You can safely edit the API key here. Make sure to include the ' before and after the key headlineApiKey = '21d8a80b3d6b444998a80b3d6b1449d3' @@ -312,40 +313,43 @@ def getAlerts(location): # TODO: This should be converted into a function so it works better with async, that way we're not getting hung up on that time.sleep() call. - -n = 0 -while n==0: - #Start our XML File - with open(path + '\\Output\\BERecord.xml', "w") as e: - e.write('') - e.close() - for i in interestList: - getAlerts(i) - #Close our XML File - with open(path + '\\Output\\BERecord.xml', "a") as d: - d.write('') - d.close() - dom = xml.dom.minidom.parse(path + '\\Output\\BERecord.xml') - pretty_xml_as_string = dom.toprettyxml(indent = " ") +# n = 0 +# while n==0: +# #Start our XML File +# with open(path + '\\Output\\BERecord.xml', "w") as e: +# e.write('') +# e.close() +# for i in interestList: +# getAlerts(i) +# #Close our XML File +# with open(path + '\\Output\\BERecord.xml', "a") as d: +# d.write('') +# d.close() +# dom = xml.dom.minidom.parse(path + '\\Output\\BERecord.xml') +# pretty_xml_as_string = dom.toprettyxml(indent = " ") - with open(path + '\\Output\\BERecord.i2m', "w") as h: - h.write(pretty_xml_as_string[23:]) - h.close() - if k > 0: - with open(path + '\\Output\\BERecord.i2m', 'rb') as f_in: - with gzip.open(path + '\\Output\\BERecord.gz', 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) +# with open(path + '\\Output\\BERecord.i2m', "w") as h: +# h.write(pretty_xml_as_string[23:]) +# h.close() +# if k > 0: +# with open(path + '\\Output\\BERecord.i2m', 'rb') as f_in: +# with gzip.open(path + '\\Output\\BERecord.gz', 'wb') as f_out: +# shutil.copyfileobj(f_in, f_out) - files = [] - commands = [] - gZipFile = path + '\\Output\\BERecord.gz' - files.append(gZipFile) - command = commands.append('') - bit.sendFile(files, commands, 1, 0) - os.remove(gZipFile) - k = 0 - os.remove(path + '\\Output\\BERecord.xml') - #os.remove(path + '\\Output\\BERecord.i2m') +# files = [] +# commands = [] +# gZipFile = path + '\\Output\\BERecord.gz' +# files.append(gZipFile) +# command = commands.append('') +# bit.sendFile(files, commands, 1, 0) +# os.remove(gZipFile) +# k = 0 +# os.remove(path + '\\Output\\BERecord.xml') +# #os.remove(path + '\\Output\\BERecord.i2m') - print('Will sleep for 60 seconds...\n') - time.sleep(60) \ No newline at end of file +# print('Will sleep for 60 seconds...\n') +# time.sleep(60) + + +def makeBERecord(): + pass \ No newline at end of file diff --git a/recordGenerators/Breathing.py b/recordGenerators/Breathing.py index fa6cf97..8797427 100644 --- a/recordGenerators/Breathing.py +++ b/recordGenerators/Breathing.py @@ -5,6 +5,7 @@ import uuid import os import shutil import xml.dom.minidom +import logging,coloredlogs sys.path.append("./py2lib") sys.path.append("./Util") @@ -13,6 +14,8 @@ import bit import MachineProductCfg as MPC import LFRecord as LFR +l = logging.getLogger(__name__) +coloredlogs.install() coopIds = [] geocodes = [] @@ -38,7 +41,7 @@ def getData(coopId, geocode): newData = data[63:-26] - print('[BREATHING] Gathering data for location id ' + coopId) + l.debug('Gathering data for location id ' + coopId) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(coopId) + '' @@ -48,6 +51,7 @@ def getData(coopId, geocode): def makeDataFile(): + l.info("Writing a Breathing forecast record.") header = '' footer = '' diff --git a/recordGenerators/CurrentObservations.py b/recordGenerators/CurrentObservations.py index 7f34139..0f4286a 100644 --- a/recordGenerators/CurrentObservations.py +++ b/recordGenerators/CurrentObservations.py @@ -5,6 +5,7 @@ import uuid import os import shutil import xml.dom.minidom +import logging,coloredlogs import sys sys.path.append("./py2lib") @@ -14,6 +15,8 @@ import bit import MachineProductCfg as MPC import LFRecord as LFR +l = logging.getLogger(__name__) +coloredlogs.install() tecciId = [] zipCodes = [] @@ -37,7 +40,7 @@ def getData(tecci, zipCode): newData = data[67:-30] - print('[CURRENT CONDITIONS] Gathering data for location id ' + tecci) + l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' @@ -47,6 +50,7 @@ def getData(tecci, zipCode): f.close() def makeDataFile(): + l.info("Writing a CurrentObservations record.") header = '' footer = '' diff --git a/recordGenerators/DailyForecast.py b/recordGenerators/DailyForecast.py index 0440c2d..97d42c8 100644 --- a/recordGenerators/DailyForecast.py +++ b/recordGenerators/DailyForecast.py @@ -5,6 +5,7 @@ import uuid import os import shutil import xml.dom.minidom +import logging,coloredlogs sys.path.append("./py2lib") sys.path.append("./Util") @@ -13,6 +14,8 @@ import bit import MachineProductCfg as MPC import LFRecord as LFR +l = logging.getLogger(__name__) +coloredlogs.install() tecciId = [] zipCodes = [] @@ -35,7 +38,7 @@ def getData(tecci, zipCode): newData = data[61:-24] - print('[DAILY FORECAST] Gathering data for location id ' + tecci) + l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' @@ -45,6 +48,7 @@ def getData(tecci, zipCode): def makeDataFile(): + l.info("Writing a DailyForecast record.") header = '' footer = '' diff --git a/recordGenerators/HourlyForecast.py b/recordGenerators/HourlyForecast.py index 040a3ea..f0f6707 100644 --- a/recordGenerators/HourlyForecast.py +++ b/recordGenerators/HourlyForecast.py @@ -4,6 +4,7 @@ import uuid import os import shutil import xml.dom.minidom +import logging,coloredlogs import sys sys.path.append("./py2lib") @@ -13,6 +14,8 @@ import bit import MachineProductCfg as MPC import LFRecord as LFR +l = logging.getLogger(__name__) +coloredlogs.install() tecciId = [] zipCodes = [] @@ -35,7 +38,7 @@ def getData(tecci, zipCode): newData = data[48:-11] - print('[HOURLY FORECAST] Gathering data for location id ' + tecci) + l.debug('Gathering data for location id ' + tecci) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(tecci) + '' @@ -44,6 +47,7 @@ def getData(tecci, zipCode): f.close() def makeDataFile(): + l.info("Writing an HourlyForecast record.") header = '' footer = '' diff --git a/recordGenerators/PollenForecast.py b/recordGenerators/PollenForecast.py index 0f99aed..d22f65b 100644 --- a/recordGenerators/PollenForecast.py +++ b/recordGenerators/PollenForecast.py @@ -5,6 +5,7 @@ import uuid import os import shutil import xml.dom.minidom +import logging, coloredlogs sys.path.append("./py2lib") sys.path.append("./Util") @@ -14,6 +15,9 @@ import MachineProductCfg as MPC import LFRecord as LFR +l = logging.getLogger(__name__) +coloredlogs.install() + pollenIds = [] geocodes = [] @@ -38,7 +42,7 @@ def getData(pollenId, geocode): newData = data[63:-26] - print('[POLLEN FORECAST] Gathering data for location id ' + pollenId) + l.debug('Gathering data for location id ' + pollenId) #Write to .i2m file i2Doc = '' + '' + newData + '' + str(pollenId) + '' @@ -48,6 +52,7 @@ def getData(pollenId, geocode): def makeDataFile(): + l.info("Writing a PollenForecast record.") header = '' footer = ''