Use logging and coloredlogs instead of printing

This commit is contained in:
April 2022-10-17 22:17:38 -07:00
parent 7a32af77aa
commit ec9268deae
No known key found for this signature in database
GPG Key ID: 17A9A017FAA4DE5E
12 changed files with 193 additions and 72 deletions

14
main.py
View File

@ -1,19 +1,25 @@
import asyncio
# from re import A
import logging,coloredlogs
from recordGenerators import DailyForecast,CurrentObservations,HourlyForecast,AirQuality,AirportDelays,PollenForecast,Breathing
l = logging.getLogger(__name__)
coloredlogs.install(logger=l)
"""
CurrentConditions: Every 5 minutes
Daily Forecasts, Hourlies, etc: 60 minutes
Alerts: 5 minutes
"""
l.info("Starting i2RecordCollector")
l.info("Developed by mewtek32, Floppaa, and Goldblaze")
print("i2MessageEncoder-Python\nDeveloped by mewtek\nData record generators by Floppaa & Goldblaze")
async def FiveMinUpdaters():
while True:
CurrentObservations.makeDataFile()
print("Sleeping for 5 minutes...")
l.debug("Sleeping for 5 minutes...")
await asyncio.sleep(300)
@ -25,7 +31,7 @@ async def HourUpdaters():
PollenForecast.makeDataFile()
AirportDelays.writeData()
Breathing.makeDataFile()
print("Sleeping for an hour...")
l.debug("Sleeping for an hour...")
await asyncio.sleep(3600)
loop = asyncio.get_event_loop()

View File

@ -5,6 +5,10 @@ import struct
import binascii
import math
import time
import logging,coloredlogs
l = logging.getLogger(__name__)
coloredlogs.install()
MCAST_GRP = '224.1.1.77'
MCAST_IF = '127.0.0.1'
@ -23,7 +27,7 @@ def sendFile(files, commands, numSgmts, Pri):
elif Pri == 1:
MCAST_PORT = 7788
else:
print("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...")
l.critical("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...")
exit()
#Get the next message ID
with open('C:\\Clips\\msgId.txt', "r") as f:
@ -37,9 +41,9 @@ def sendFile(files, commands, numSgmts, Pri):
h.close()
segnmNum = 0
if Pri == 0:
print("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
l.info("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
elif Pri == 1:
print("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
l.info("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
startFlag = False
for x, y in zip(files, commands):
@ -85,7 +89,7 @@ def sendFile(files, commands, numSgmts, Pri):
conn.sendto(packetHeader + fec + data + theNull, (MCAST_GRP, MCAST_PORT))
else:
conn.sendto(packetHeader + fec + data, (MCAST_GRP, MCAST_PORT))
print(packet_count)
l.debug(packet_count)
packet_count += 1
j += 1
@ -115,7 +119,7 @@ def sendCommand(command, Pri, msgNum = None):
elif Pri == 1:
MCAST_PORT = 7788
else:
print("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...")
l.critical("Invalid Priority Flag. 0 = Routine Message 1 = High Priority Message\n\nScript will now terminate...")
exit()
#Get the next message ID
with open('C:\\Clips\\msgId.txt', "r") as f:
@ -129,9 +133,9 @@ def sendCommand(command, Pri, msgNum = None):
h.close()
segnmNum = 0
if Pri == 0:
print("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
l.info("Sending Routine Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
elif Pri == 1:
print("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
l.info("Sending High Priority Msg-" + str(msgNum) + " on UDP " + MCAST_GRP + " " + str(MCAST_PORT) + "....")
startFlag = False
for x in command:
@ -181,7 +185,7 @@ def sendCommand(command, Pri, msgNum = None):
conn.sendto(packetHeader + fec + data + theNull, (MCAST_GRP, MCAST_PORT))
else:
conn.sendto(packetHeader + fec + data, (MCAST_GRP, MCAST_PORT))
print(packet_count)
l.debug(packet_count)
packet_count += 1
j += 1

View File

@ -7,6 +7,7 @@ import aiohttp
import json
import time as epochTime
import requests
import logging,coloredlogs
from RadarProcessor import *
from os import path, mkdir, listdir, remove, cpu_count
@ -18,6 +19,9 @@ from wand.color import Color
radarType = "Radar-US"
l = logging.getLogger(__name__)
coloredlogs.install()
upperLeftX,upperLeftY,lowerRightX,lowerRightY = 0,0,0,0
xStart,xEnd,yStart,yEnd = 0,0,0,0
imgW = 0
@ -29,7 +33,7 @@ import bit
async def getValidTimestamps(boundaries:ImageBoundaries) -> list:
"""Gets all valid UNIX timestamps for the TWCRadarMosaic product """
print("Getting timestamps for the radar..")
l.info("Getting timestamps for the radar..")
times = []
async with aiohttp.ClientSession() as session:
@ -44,12 +48,12 @@ async def getValidTimestamps(boundaries:ImageBoundaries) -> list:
# Don't add frames that aren't at the correct interval
if (time % boundaries.ImageInterval != 0):
print(f"Ignoring {time} -- Not at the correct frame interval.")
l.debug(f"Ignoring {time} -- Not at the correct frame interval.")
continue
# Don't add frames that are expired
if (time < (datetime.utcnow().timestamp() - epochTime.time()) / 1000 - boundaries.Expiration):
print(f"Ignoring {time} -- Expired.")
l.debug(f"Ignoring {time} -- Expired.")
continue
times.append(time)
@ -63,13 +67,13 @@ def downloadRadarTile(url, p, fn):
# Make the path if it doesn't exist
if exists(f"tiles/output/{ts}.tiff"):
print("Not downloading tiles for timestamp " + str(ts) + " since a frame for it already exists." )
l.debug("Not downloading tiles for timestamp " + str(ts) + " since a frame for it already exists." )
download = False
if not path.exists(p):
mkdir(p)
print(f"Download {ts}")
l.debug(f"Download {ts}")
if exists(f"{p}/{fn}"):
print(f"Not downloading new tiles for {ts} as they already exist.")
l.debug(f"Not downloading new tiles for {ts} as they already exist.")
download = False
if (img.status_code == 200 and download):
@ -77,7 +81,7 @@ def downloadRadarTile(url, p, fn):
for data in img:
tile.write(data)
elif (img.status_code != 200):
print("ERROR DOWNLOADING " + p + "\nSTATUS CODE " + str(img.status_code))
l.error("ERROR DOWNLOADING " + p + "\nSTATUS CODE " + str(img.status_code))
elif (download == False):
pass
@ -206,6 +210,7 @@ def getTime(timestamp) -> str:
async def makeRadarImages():
""" Creates proper radar frames for the i2 """
l.info("Downloading frames for the Regional Radar...")
combinedCoordinates = []
@ -221,7 +226,7 @@ async def makeRadarImages():
# Get rid of invalid radar frames
for i in listdir('tiles/output'):
if i.split('.')[0] not in [str(x) for x in times] and i != "Thumbs.db":
print(f"Deleting {i} as it is no longer valid.")
l.debug(f"Deleting {i} as it is no longer valid.")
remove("tiles/output/" + i)
# Collect coordinates for the frame tiles
@ -242,7 +247,7 @@ async def makeRadarImages():
paths.append(f"tiles/{times[i]}")
filenames.append(f"{times[i]}_{combinedCoordinates[c].x}_{combinedCoordinates[c].y}.png")
print(len(urls))
l.debug(len(urls))
if len(urls) != 0 and len(urls) >= 6:
with Pool(cpu_count() - 1) as p:
p.starmap(downloadRadarTile, zip(urls, paths, filenames))
@ -254,7 +259,7 @@ async def makeRadarImages():
p.close()
p.join()
elif len(urls) == 0:
print("No new radar frames need to be downloaded.")
l.info("No new radar frames need to be downloaded.")
return
# Stitch them all together!
@ -270,7 +275,7 @@ async def makeRadarImages():
# Stitch the frames together
for i in range(0, len(imgsToGenerate)):
if not exists(F"tiles/output/{times[i]}.tiff"):
print(f"Generate frame for {times[i]}")
l.debug(f"Generate frame for {times[i]}")
for c in combinedCoordinates:
path = f"tiles/{times[i]}/{times[i]}_{c.x}_{c.y}.png"
@ -290,7 +295,7 @@ async def makeRadarImages():
# Composite images for the i2
for img in framesToComposite:
print("Attempting to composite " + img)
l.debug("Attempting to composite " + img)
# Crop the radar images something that the i2 will actually take
img_raw = wandImage(filename=img)
@ -315,6 +320,8 @@ async def makeRadarImages():
bit.sendFile([finished[i]], [commands[i]], 1, 0)
l.info("Downloaded and sent Regional Radar frames!")
# print(getTime(1665880800))

View File

@ -0,0 +1,69 @@
import asyncio
import aiohttp
import time as epochTime
import datetime
import requests
from RadarProcessor import *
from os import mkdir, path
from genericpath import exists
upperLeftX,upperLeftY,lowerRightX,lowerRightY = 0,0,0,0
xStart,xEnd,yStart,yEnd = 0,0,0,0
imgW = 0
imgH = 0
async def getValidTimestamps(boundaries:ImageBoundaries) -> list:
"""Gets all valid UNIX timestamps for the TWCRadarMosaic product """
print("Getting timestamps for the radar..")
times = []
async with aiohttp.ClientSession() as session:
url = "https://api.weather.com/v3/TileServer/series/productSet?apiKey=21d8a80b3d6b444998a80b3d6b1449d3&filter=twcRadarMosaic"
async with session.get(url) as r:
response = await r.json()
for t in range(0, len(response['seriesInfo']['twcRadarMosaic']['series'])):
if (t <= 35):
time = response['seriesInfo']['twcRadarMosaic']['series'][t]['ts']
# Don't add frames that aren't at the correct interval
if (time % boundaries.ImageInterval != 0):
print(f"Ignoring {time} -- Not at the correct frame interval.")
continue
# Don't add frames that are expired
if (time < (datetime.utcnow().timestamp() - epochTime.time()) / 1000 - boundaries.Expiration):
print(f"Ignoring {time} -- Expired.")
continue
times.append(time)
return times
def downloadRadarTile(url, p, fn):
img = requests.get(url, stream=True)
ts = fn.split("_")[0]
download = True
# Make the path if it doesn't exist
if exists(f"tiles/output/{ts}.tiff"):
print("Not downloading tiles for timestamp " + str(ts) + " since a frame for it already exists." )
download = False
if not path.exists(p):
mkdir(p)
print(f"Download {ts}")
if exists(f"{p}/{fn}"):
print(f"Not downloading new tiles for {ts} as they already exist.")
download = False
if (img.status_code == 200 and download):
with open(f'{p}/{fn}', 'wb') as tile:
for data in img:
tile.write(data)
elif (img.status_code != 200):
print("ERROR DOWNLOADING " + p + "\nSTATUS CODE " + str(img.status_code))
elif (download == False):
pass

View File

@ -3,6 +3,10 @@ import gzip
import os
import shutil
import xml.dom.minidom
import logging,coloredlogs
l = logging.getLogger(__name__)
coloredlogs.install()
import sys
sys.path.append("./py2lib")
@ -44,9 +48,9 @@ def writeData():
for i in epaIds:
if i == None:
print(f"No EPA ID found for location -- Skipping.")
l.debug(f"No EPA ID found for location -- Skipping.")
else:
print(f"EPA ID found for location! Writing data for Air Quality.")
l.debug(f"EPA ID found for location! Writing data for Air Quality.")
workingEpaIds.append(i)
useData = True
@ -54,6 +58,7 @@ def writeData():
# Check to see if we even have EPA ids, as some areas don't have air quality reports
if (useData):
try:
l.info("Writing an AirQuality record.")
header = '<Data type="AirQuality">'
footer = "</Data>"
@ -90,9 +95,10 @@ def writeData():
os.remove("D:\\AirQuality.i2m")
os.remove("D:\\AirQuality.gz")
except Exception as e:
print("AirQuality failed to write, problably was expired and was sticking around in the IBM api.")
l.error("DO NOT REPORT THE ERROR BELOW")
l.error("Failed to write an AirQuality record.")
else:
print("Ignoring AirQuality data collection -- No working EPA Ids.")
l.info("Not writing an AirQuality record due to a lack of working EPA ids.")

View File

@ -3,6 +3,7 @@ import gzip
import os
import shutil
import xml.dom.minidom
import logging,coloredlogs
import sys
sys.path.append("./py2lib")
@ -12,6 +13,9 @@ import bit
import MachineProductCfg as MPC
import LFRecord as LFR
l = logging.getLogger(__name__)
coloredlogs.install()
locationIds = []
zipCodes = []
airports = []
@ -35,7 +39,6 @@ def getData(airport):
# Write to i2doc file
i2Doc = f'<AirportDelays id="000000000" locationKey="{airport}" isWxScan="0">' + '' + newData + f'<clientKey>{airport}</clientKey></AirportDelays>'
print(f"[AIRPORT DELAYS] Writing airport delay data for {airport}")
f = open("D:\\AirportDelays.i2m", 'a')
f.write(i2Doc)
@ -50,13 +53,14 @@ def writeData():
res = requests.get(f"https://api.weather.com/v1/airportcode/{x}/airport/delays.xml?language=en-US&apiKey={apiKey}")
if (res.status_code != 200):
print(f"[AIRPORT DELAYS] No delays for {x} found, skipping..")
l.debug(f"[AIRPORT DELAYS] No delays for {x} found, skipping..")
else:
airportsWithDelays.append(x)
print(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..")
l.debug(f"[AIRPORT DELAYS] {x} has a delay! Writing a file..")
useData = True
if (useData):
l.info("Writing an AirportDelays record.")
header = '<Data type="AirportDelays">'
footer = "</Data>"
@ -93,4 +97,4 @@ def writeData():
os.remove("D:\\AirportDelays.i2m")
os.remove("D:\\AirportDelays.gz")
else:
print("[AIRPORT DELAYS] Not writing AirportDelays -- Either no delays found, or the API is broken.")
l.info("No airport delays found.")

View File

@ -2,6 +2,7 @@ import requests
import json
import os
from datetime import datetime,timedelta
from Util.MachineProductCfg import getZones
import time
import pytz
import xml.dom.minidom
@ -10,7 +11,7 @@ import gzip
import py2Lib.bit as bit
#Zones/Counties to fetch alerts for
interestList = ['FLZ151', 'FLC057', 'FLZ149', 'FLZ249', 'FLC101'] # TODO: Grab these automatically from MachineProductCfg.xml
zones = getZones()
#You can safely edit the API key here. Make sure to include the ' before and after the key
headlineApiKey = '21d8a80b3d6b444998a80b3d6b1449d3'
@ -312,40 +313,43 @@ def getAlerts(location):
# TODO: This should be converted into a function so it works better with async, that way we're not getting hung up on that time.sleep() call.
n = 0
while n==0:
#Start our XML File
with open(path + '\\Output\\BERecord.xml', "w") as e:
e.write('<Data type="BERecord">')
e.close()
for i in interestList:
getAlerts(i)
#Close our XML File
with open(path + '\\Output\\BERecord.xml', "a") as d:
d.write('</Data>')
d.close()
dom = xml.dom.minidom.parse(path + '\\Output\\BERecord.xml')
pretty_xml_as_string = dom.toprettyxml(indent = " ")
# n = 0
# while n==0:
# #Start our XML File
# with open(path + '\\Output\\BERecord.xml', "w") as e:
# e.write('<Data type="BERecord">')
# e.close()
# for i in interestList:
# getAlerts(i)
# #Close our XML File
# with open(path + '\\Output\\BERecord.xml', "a") as d:
# d.write('</Data>')
# d.close()
# dom = xml.dom.minidom.parse(path + '\\Output\\BERecord.xml')
# pretty_xml_as_string = dom.toprettyxml(indent = " ")
with open(path + '\\Output\\BERecord.i2m', "w") as h:
h.write(pretty_xml_as_string[23:])
h.close()
if k > 0:
with open(path + '\\Output\\BERecord.i2m', 'rb') as f_in:
with gzip.open(path + '\\Output\\BERecord.gz', 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
# with open(path + '\\Output\\BERecord.i2m', "w") as h:
# h.write(pretty_xml_as_string[23:])
# h.close()
# if k > 0:
# with open(path + '\\Output\\BERecord.i2m', 'rb') as f_in:
# with gzip.open(path + '\\Output\\BERecord.gz', 'wb') as f_out:
# shutil.copyfileobj(f_in, f_out)
files = []
commands = []
gZipFile = path + '\\Output\\BERecord.gz'
files.append(gZipFile)
command = commands.append('<MSG><Exec workRequest="storeData(File={0},QGROUP=__BERecord__,Feed=BERecord)" /><GzipCompressedMsg fname="BERecord" /></MSG>')
bit.sendFile(files, commands, 1, 0)
os.remove(gZipFile)
k = 0
os.remove(path + '\\Output\\BERecord.xml')
#os.remove(path + '\\Output\\BERecord.i2m')
# files = []
# commands = []
# gZipFile = path + '\\Output\\BERecord.gz'
# files.append(gZipFile)
# command = commands.append('<MSG><Exec workRequest="storeData(File={0},QGROUP=__BERecord__,Feed=BERecord)" /><GzipCompressedMsg fname="BERecord" /></MSG>')
# bit.sendFile(files, commands, 1, 0)
# os.remove(gZipFile)
# k = 0
# os.remove(path + '\\Output\\BERecord.xml')
# #os.remove(path + '\\Output\\BERecord.i2m')
print('Will sleep for 60 seconds...\n')
time.sleep(60)
# print('Will sleep for 60 seconds...\n')
# time.sleep(60)
def makeBERecord():
pass

View File

@ -5,6 +5,7 @@ import uuid
import os
import shutil
import xml.dom.minidom
import logging,coloredlogs
sys.path.append("./py2lib")
sys.path.append("./Util")
@ -13,6 +14,8 @@ import bit
import MachineProductCfg as MPC
import LFRecord as LFR
l = logging.getLogger(__name__)
coloredlogs.install()
coopIds = []
geocodes = []
@ -38,7 +41,7 @@ def getData(coopId, geocode):
newData = data[63:-26]
print('[BREATHING] Gathering data for location id ' + coopId)
l.debug('Gathering data for location id ' + coopId)
#Write to .i2m file
i2Doc = '<Breathing id="000000000" locationKey="' + str(coopId) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(coopId) + '</clientKey></Breathing>'
@ -48,6 +51,7 @@ def getData(coopId, geocode):
def makeDataFile():
l.info("Writing a Breathing forecast record.")
header = '<Data type="Breathing">'
footer = '</Data>'

View File

@ -5,6 +5,7 @@ import uuid
import os
import shutil
import xml.dom.minidom
import logging,coloredlogs
import sys
sys.path.append("./py2lib")
@ -14,6 +15,8 @@ import bit
import MachineProductCfg as MPC
import LFRecord as LFR
l = logging.getLogger(__name__)
coloredlogs.install()
tecciId = []
zipCodes = []
@ -37,7 +40,7 @@ def getData(tecci, zipCode):
newData = data[67:-30]
print('[CURRENT CONDITIONS] Gathering data for location id ' + tecci)
l.debug('Gathering data for location id ' + tecci)
#Write to .i2m file
i2Doc = '<CurrentObservations id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></CurrentObservations>'
@ -47,6 +50,7 @@ def getData(tecci, zipCode):
f.close()
def makeDataFile():
l.info("Writing a CurrentObservations record.")
header = '<Data type="CurrentObservations">'
footer = '</Data>'

View File

@ -5,6 +5,7 @@ import uuid
import os
import shutil
import xml.dom.minidom
import logging,coloredlogs
sys.path.append("./py2lib")
sys.path.append("./Util")
@ -13,6 +14,8 @@ import bit
import MachineProductCfg as MPC
import LFRecord as LFR
l = logging.getLogger(__name__)
coloredlogs.install()
tecciId = []
zipCodes = []
@ -35,7 +38,7 @@ def getData(tecci, zipCode):
newData = data[61:-24]
print('[DAILY FORECAST] Gathering data for location id ' + tecci)
l.debug('Gathering data for location id ' + tecci)
#Write to .i2m file
i2Doc = '<DailyForecast id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></DailyForecast>'
@ -45,6 +48,7 @@ def getData(tecci, zipCode):
def makeDataFile():
l.info("Writing a DailyForecast record.")
header = '<Data type="DailyForecast">'
footer = '</Data>'

View File

@ -4,6 +4,7 @@ import uuid
import os
import shutil
import xml.dom.minidom
import logging,coloredlogs
import sys
sys.path.append("./py2lib")
@ -13,6 +14,8 @@ import bit
import MachineProductCfg as MPC
import LFRecord as LFR
l = logging.getLogger(__name__)
coloredlogs.install()
tecciId = []
zipCodes = []
@ -35,7 +38,7 @@ def getData(tecci, zipCode):
newData = data[48:-11]
print('[HOURLY FORECAST] Gathering data for location id ' + tecci)
l.debug('Gathering data for location id ' + tecci)
#Write to .i2m file
i2Doc = '<HourlyForecast id="000000000" locationKey="' + str(tecci) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(tecci) + '</clientKey></HourlyForecast>'
@ -44,6 +47,7 @@ def getData(tecci, zipCode):
f.close()
def makeDataFile():
l.info("Writing an HourlyForecast record.")
header = '<Data type="HourlyForecast">'
footer = '</Data>'

View File

@ -5,6 +5,7 @@ import uuid
import os
import shutil
import xml.dom.minidom
import logging, coloredlogs
sys.path.append("./py2lib")
sys.path.append("./Util")
@ -14,6 +15,9 @@ import MachineProductCfg as MPC
import LFRecord as LFR
l = logging.getLogger(__name__)
coloredlogs.install()
pollenIds = []
geocodes = []
@ -38,7 +42,7 @@ def getData(pollenId, geocode):
newData = data[63:-26]
print('[POLLEN FORECAST] Gathering data for location id ' + pollenId)
l.debug('Gathering data for location id ' + pollenId)
#Write to .i2m file
i2Doc = '<PollenForecast id="000000000" locationKey="' + str(pollenId) + '" isWxscan="0">' + '' + newData + '<clientKey>' + str(pollenId) + '</clientKey></PollenForecast>'
@ -48,6 +52,7 @@ def getData(pollenId, geocode):
def makeDataFile():
l.info("Writing a PollenForecast record.")
header = '<Data type="PollenForecast">'
footer = '</Data>'