Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
ae8c8c7b90 | ||
|
|
2487310eb1 | ||
|
|
e4221baf7e | ||
|
|
0b06939718 | ||
|
|
029efbccab | ||
|
|
ba154b324a | ||
|
|
cdb303825d | ||
|
|
e4f0d705c6 | ||
|
|
ccec7d8416 |
12
.gitignore
vendored
12
.gitignore
vendored
@ -1,3 +1,13 @@
|
||||
CCTVCrawl*
|
||||
devscrape*
|
||||
*.log*
|
||||
OLD_devscrape*
|
||||
*.log
|
||||
TransArchive*.csv
|
||||
|
||||
# ide/py cache/settings files
|
||||
__pycache__/
|
||||
.vscode/
|
||||
|
||||
# do not sync archive feeds prod folder
|
||||
archive/
|
||||
|
||||
|
||||
18
README.md
18
README.md
@ -1,2 +1,20 @@
|
||||
# TranstarArchive
|
||||
Python application intended to store an archive of live camera feeds from Houston Transtar
|
||||
|
||||
## requirements
|
||||
python 3.6 or later
|
||||
|
||||
the requests and wget Python modules, installed with pip (if others aren't found on your system, they can be easily viewed in the source code c:)
|
||||
|
||||
## what does this program do?
|
||||
retrives images from live and valid camera image feeds from Houston Transtar. normally these can be viewed on their website [here](https://traffic.houstontranstar.org/cctv/transtar/), or a very very lightweight version (intended for mobile or older devices, that i prefer) can be found [here](http://traffic.houstontranstar.org/mobile/transtar_cctv.aspx), stores them in a folder/local archive on the system you run it on
|
||||
|
||||
## why?
|
||||
i have no idea
|
||||
|
||||
## arguments
|
||||
by default and with no args provided TranstarArchive will create required folders if necessary and bring you to the main menu. this is useful for archiving once, however below can be more useful if you want to continuously archive.
|
||||
|
||||
-archive will start archiving feeds with no extra prompts. use a looping script provided (for windows as of 9/8/22) to continuously loop the process.
|
||||
|
||||
-testarg is self explanatory. tests if TranstarArchive will accept arguments
|
||||
|
||||
13
TranstarArchive_Loop.bat
Normal file
13
TranstarArchive_Loop.bat
Normal file
@ -0,0 +1,13 @@
|
||||
@echo off
|
||||
title TranstarArchive Looping Test (for Windows)
|
||||
echo %cd%
|
||||
set /p confirm=This will create a loop for archiving files from transtar, delayed every few minutes, are you sure?
|
||||
if %confirm% == yes goto loop
|
||||
if %confirm% == no goto :eof
|
||||
goto :eof
|
||||
|
||||
:loop
|
||||
echo Starting to loop @ %date% %time%
|
||||
python.exe %cd%\main.py -archive
|
||||
timeout 17
|
||||
goto loop
|
||||
165
gifcombine.py
Normal file
165
gifcombine.py
Normal file
@ -0,0 +1,165 @@
|
||||
import imageio
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import time
|
||||
|
||||
# TranstarArchive Image GIF Combine PY
|
||||
# author: iRaven (https://iravenhome.net)
|
||||
# Started 11/22/22.
|
||||
|
||||
# version="2022.11.22"
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(filename="TranstarArchive_gifCombine.log",
|
||||
format='%(asctime)s %(levelname)s %(message)s',
|
||||
filemode='a')
|
||||
log = logging.getLogger()
|
||||
# log.setLevel(logging.DEBUG) # use this for debugging
|
||||
log.setLevel(logging.INFO) # use this for production
|
||||
|
||||
arglist = sys.argv[1:]
|
||||
|
||||
#time -- if this will ever be used? logging takes care of this
|
||||
initcurtime = time.localtime()
|
||||
curtime = time.strftime("%H-%M-%S", initcurtime)
|
||||
curdate = (f'{initcurtime.tm_year}-{initcurtime.tm_mon}-{initcurtime.tm_mday}')
|
||||
|
||||
global imagespath
|
||||
global dateinput
|
||||
|
||||
# valid feeds list to iterate thru
|
||||
# TODO: make this user supplied for older folders maybe
|
||||
vftxt = open("valid_feed_list.txt", "r")
|
||||
vfdata = vftxt.read()
|
||||
vfeeds = vfdata.split("\n")
|
||||
vftxt.close()
|
||||
|
||||
def initgifFolder(path, date): # makes sure gif folder exists for date provided in a certain path
|
||||
if not os.path.exists(f'{path}/{date}/gifs'):
|
||||
log.warning(f"gifs folder in {path}/{date} does not exist, creating it")
|
||||
os.makedirs(f"{path}/{date}/gifs")
|
||||
else:
|
||||
log.info(f"gifs folder in {path}/{date} exists, ignoring check")
|
||||
|
||||
|
||||
def gifCombine(path, date):
|
||||
# images=[]
|
||||
initgifFolder(path,date) # make sure gif folder exists for date provided
|
||||
joinedpath = os.path.join(path, date)
|
||||
# print(joinedpath) # debugging- comment this out on prod !!!
|
||||
for feednum in vfeeds:
|
||||
images=[] # list of images found to append to output gif image for each feednum (resets with nothing for each feednum)
|
||||
imagenames=[]
|
||||
for filenames in sorted(os.listdir(joinedpath)): # for each file in provided path dir
|
||||
if filenames.startswith(feednum): # if a file starts with the current feednum
|
||||
# length check!! (this bit me in the ass)
|
||||
# print(len(filenames.split('_')[0])) # debugging
|
||||
if len(filenames.split('_')[0]) == len(feednum):
|
||||
log.info(f"found {filenames} that starts with {feednum}")
|
||||
print(f"found {filenames} that starts with {feednum}")
|
||||
if filenames in imagenames: # fsakdjfjaksldfjklasdfjklakljdsxf holy shit so many checks! see if a file name has already been added to list for gif
|
||||
log.debug(f'found {filenames} already in {feednum} filename list so NOT adding to list AGAIN')
|
||||
else:
|
||||
# log.info("") # not sure what i was doing here
|
||||
#if os.path.isfile(f'{joinedpath}/gifs/{feednum}.gif'): # check if a file exists so we don't waste time with imageio reading contents
|
||||
# (TODO: may improve if ran earlier and new feeds are added?) (this is that LOL) (nope this doesn't really work)
|
||||
# log.info(f"gif file for {feednum} already exists at \'{joinedpath}/gifs/{feednum}.gif\', not creating again and moving on...")
|
||||
# print(f"gif file for {feednum} already exists at \'{joinedpath}/gifs/{feednum}.gif\', not creating again and moving on...")
|
||||
#else:
|
||||
imagenames.append(filenames) # append found file names to list for adding to imageio reader appending list
|
||||
else:
|
||||
log.debug(f'found {filenames} that starts with {feednum} HOWEVER isn\'t right length')
|
||||
else:
|
||||
log.debug(f"{filenames} does not start with {feednum}")
|
||||
|
||||
for files in imagenames:
|
||||
log.info(f"applying image {files} for {feednum} in {joinedpath}")
|
||||
print(f"applying image {files} for {feednum} in {joinedpath}")
|
||||
try:
|
||||
images.append(imageio.imread(f'{joinedpath}/{files}')) # append whatever image found that starts with current feednum to list
|
||||
#TODO: this gives a deprecated error about imread because of course it does.
|
||||
except ValueError:
|
||||
log.debug(f"a corrupted file has been detected! {images}")
|
||||
log.error(f"a ValueError has occured on {feednum}, maybe due to a corrupted .jpg file that may have been appended.")
|
||||
log.error(f"{feednum} ValueError: {joinedpath}/gifs/{feednum}.gif will not be created and ignored.")
|
||||
images.remove(files) # maybe this will remove only the file that got fucked up?
|
||||
|
||||
# log.debug(f"all images found to be saved: \n{images}") -- this generates way too much bullshit
|
||||
log.debug(f" all images to be saved in a gif: \n{imagenames}")
|
||||
log.info(f"saving {feednum} gif file with all images collected of it on {date}...")
|
||||
print(f"saving {feednum} gif file with all images collected of it on {date}...")
|
||||
log.debug(f"checking if gif for {feednum} already exists")
|
||||
if os.path.isfile(f'{joinedpath}/gifs/{feednum}.gif'): # check if a file exists so we don't create it again (TODO: may improve if ran earlier and new feeds are added?)
|
||||
log.info(f"gif file for {feednum} already exists at \'{joinedpath}/gifs/{feednum}.gif\', not creating again and moving on...")
|
||||
else: # if file doesn't exist already, make it
|
||||
imageio.mimsave(f'{joinedpath}/gifs/{feednum}.gif', images, duration=0.2) # save a gif image for each feednum in date folder provided with .2 secs between each frame
|
||||
log.info("gif should be saved and created successfully. moving on to next itieration...")
|
||||
print("gif should be saved and created successfully. moving on to next itieration...")
|
||||
|
||||
|
||||
|
||||
def MainMenu():
|
||||
log.info("main menu started")
|
||||
print("TranstarArchive gifCombine App")
|
||||
print("this program will combine any feed images archived by TranstarArchive for a specific date you provide")
|
||||
# print(" ")
|
||||
print(" ") # line breaks? LOL
|
||||
|
||||
print("first, let's start off with a path where all your images are stored, which must contain specific date folders.")
|
||||
print("normally this path is in the folder you ran this script in, /archive.")
|
||||
log.debug("prompting user for path selection choice, default or custom")
|
||||
menuin1 = input("please type Y if this is correct or N to enter a different folder path: ")
|
||||
menuin1 = menuin1.lower()
|
||||
if menuin1 == "y":
|
||||
imagespath = "./archive"
|
||||
log.info("images path was set to default /archive folder")
|
||||
elif menuin1 == "n":
|
||||
log.debug("user chose to enter a custom path to be scanned")
|
||||
menuin2 = input("please type a custom path here for gifCombine to scan: ")
|
||||
log.debug(f"user inputted {menuin2}, now checking if this path is actually a thing")
|
||||
if os.path.exists(menuin2):
|
||||
imagespath = menuin2
|
||||
log.info(f"user inputted path {menuin2} EXISTS, going on...")
|
||||
else:
|
||||
log.error(f"user inputted path {menuin2} DOES NOT EXIST, trying again...")
|
||||
print(f"the path you entered ({menuin2} couldn't be found, please try that again!")
|
||||
MainMenu() # this feels incredibly chaotic, a function calling itself LOL
|
||||
# exit() # ^ if that doesn't work or breaks things tremendously, fuck it
|
||||
else:
|
||||
exit() # fuck it if that selection wasn't y or n. not sure what to do, i'll improve this later (2022/11/22)
|
||||
# TODO: valid_feeds_list file input
|
||||
|
||||
print(" ") # line breaks? LOL
|
||||
log.debug("prompting user for date selection choice")
|
||||
print("second, you'll need to choose which date do you want to combine gifs for. ")
|
||||
log.debug(f"getting dir listing of {imagespath} for date selection")
|
||||
print("here is a listing of the directories in the path you provided earlier.")
|
||||
print(os.listdir(imagespath))
|
||||
menuin3 = input("type in the folder with the date you want gifCombine to iterate through: ")
|
||||
# TODO: formatting error correction
|
||||
dateinput = menuin3 # just declare it here
|
||||
|
||||
print(" ") # line breaks? LOL
|
||||
log.debug("prompting user for confirmation")
|
||||
log.info(f"final options before confirmation: path={imagespath}, date={dateinput}")
|
||||
print(f"before we start, you have chosen your file path as {imagespath} and the date to work through {dateinput}")
|
||||
print("this may take a while depending on how many images are to be processed.")
|
||||
menuin4 = input("type in \"startrans\" to continue and start processing: ")
|
||||
if menuin4 == "startrans":
|
||||
# go!
|
||||
print("making sure folders are set up...")
|
||||
initgifFolder(imagespath, dateinput) # make gifs folder in specced dir
|
||||
print("preparing for epic sonic rainboom...")
|
||||
print("now starting to gif combine!")
|
||||
gifCombine(imagespath, dateinput) # go bitch
|
||||
exit() # exit when done
|
||||
else:
|
||||
exit() # fuck off
|
||||
|
||||
# arguments
|
||||
# TODO: wait fuck that's not really a good idea rn lol
|
||||
|
||||
# start program
|
||||
log.info("TranstarArchive's gifCombine was STARTED!")
|
||||
MainMenu()
|
||||
63
main.py
63
main.py
@ -2,27 +2,29 @@ import logging
|
||||
import os
|
||||
import wget
|
||||
import time
|
||||
import datetime
|
||||
import sys
|
||||
import requests
|
||||
import csv
|
||||
import threading
|
||||
|
||||
|
||||
# TranstarArchive Main PY
|
||||
# author: iRaven (https://iravenhome.net)
|
||||
# Started 9/6/22.
|
||||
|
||||
#version = "0.1"
|
||||
#version = "2022.9.28"
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(filename="TransArchive.log",
|
||||
format='%(asctime)s %(message)s',
|
||||
logging.basicConfig(filename="TranstarArchive.log",
|
||||
format='%(asctime)s %(levelname)s %(message)s',
|
||||
filemode='a')
|
||||
log = logging.getLogger()
|
||||
log.setLevel(logging.DEBUG)
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
|
||||
#define local vars
|
||||
|
||||
arglist = sys.argv[1:]
|
||||
#time
|
||||
initcurtime = time.localtime()
|
||||
# currtime = (f'{initcurtime.tm_hour}-{initcurtime.tm_min}-{initcurtime.tm_sec}')
|
||||
@ -33,7 +35,7 @@ csvfile = (f"TransArchive_{curdate}.csv") # mainly for debugging but to show sta
|
||||
csvfields = ['time','camera_id', 'status'] # ^
|
||||
|
||||
# very important var: where we get everything
|
||||
cctvurl = (f'https://www.houstontranstar.org/snapshots/cctv/') # Last updated Sep-2022
|
||||
cctvurl = (f'https://www.houstontranstar.org/snapshots/cctv/') # Last updated Sep-6-2022
|
||||
# very important var: list of valid camera feeds generated by devs (likely included and updated in repo)
|
||||
vftxt = open("valid_feed_list.txt", "r")
|
||||
vfdata = vftxt.read()
|
||||
@ -45,8 +47,12 @@ vftxt.close()
|
||||
|
||||
######## Core functions
|
||||
# initialize folder
|
||||
def initFolder():
|
||||
log.info(f'TranstarArchive was started')
|
||||
def initFolder(startmode):
|
||||
if startmode == "script":
|
||||
log.info(f'TranstarArchive was likely started non-standalone with a script or from a command line')
|
||||
else:
|
||||
log.info(f'TranstarArchive was started')
|
||||
|
||||
if not os.path.exists("archive/"):
|
||||
log.info("archive folder doesn't exist. creating it")
|
||||
os.makedirs("archive/")
|
||||
@ -54,7 +60,7 @@ def initFolder():
|
||||
log.warning("archive folder for todays date doesn't exist (expected LOL), creating it")
|
||||
os.makedirs(f"archive/{curdate}")
|
||||
elif not os.path.exists(f"TransstarArchive_{curdate}.csv"):
|
||||
log.error("csv file doesn't exist. creating it for today")
|
||||
log.warning("csv file doesn't exist. creating it for today")
|
||||
with open(csvfile, 'a') as listfile:
|
||||
csvwrite = csv.writer(listfile)
|
||||
csvwrite.writerow(csvfields)
|
||||
@ -73,7 +79,9 @@ def imageDownload(url):
|
||||
csvvalid = [curtime,feed,'Success']
|
||||
print(f'Valid camera feed is being archived: {feed}\n')
|
||||
log.info(f'Valid camera feed is being archived: {feed}')
|
||||
wget.download(rq.url,f'archive/{curdate}/{feed}_{curtime}.jpg')
|
||||
# wget.download(rq.url,f'archive/{curdate}/{feed}_{curtime}.jpg')
|
||||
# new method - using requests module only (10.20.22)
|
||||
open(f'archive/{curdate}/{feed}_{curtime}.jpg', 'wb').write(rq.content) # ^
|
||||
with open(csvfile, 'a') as listfile:
|
||||
csvwrite = csv.writer(listfile)
|
||||
csvwrite.writerow(csvvalid)
|
||||
@ -86,7 +94,7 @@ def imageDownload(url):
|
||||
# for vals in invlog:
|
||||
# invlog.write(f'invalid feeds found at archive {curdate} at {curtime}:\n{vals}\n#### END END END ####\n either transstar is having issues or has reconfigured feed numbers, open an issue in the repo: https://github.com/iraven4522/TranstarArchive\n')
|
||||
|
||||
# loops the download function every like 3 or so mins
|
||||
# loops the download function every like 3 or so mins (may not work?)
|
||||
def imgDownloadLoop(secs):
|
||||
log.info(f'delay loop was called for {secs}, starting now')
|
||||
threading.Timer(f'{secs}.0', imgDownloadLoop).start()
|
||||
@ -106,13 +114,36 @@ def MainMenu():
|
||||
menuin1 = input("If you want to continue please type the phrase \"startrans\" (without quotes) or type no: ")
|
||||
if menuin1 == "startrans":
|
||||
imageDownload(cctvurl)
|
||||
log.info("returned to menu for delay")
|
||||
imgDownloadLoop(195)
|
||||
# log.info("returned to menu for delay")
|
||||
# imgDownloadLoop(195)
|
||||
else:
|
||||
exit()
|
||||
else:
|
||||
exit()
|
||||
|
||||
# Run the shit (finally)
|
||||
initFolder() # check folder and csv file first (send log start message)
|
||||
MainMenu()
|
||||
# Run the shit (finally) -- phased out due to arguments
|
||||
# initFolder() # check folder and csv file first (send log start message)
|
||||
# MainMenu()
|
||||
|
||||
# Arguments
|
||||
for arg in arglist:
|
||||
# print(arglist)
|
||||
if arg in ("-archive"):
|
||||
# print("Archive was passed") # debugging
|
||||
initFolder("script")
|
||||
imageDownload(cctvurl)
|
||||
elif arg in ("-testarg"): # debugging, left to test arguments
|
||||
print("test arg was passed")
|
||||
log.debug("Test arg was passed -- not running")
|
||||
else: # unrecognized arguments are passed (which is oki)
|
||||
# log.debug("no args were passed!")
|
||||
# log.debug("unrecognized args were passed!")
|
||||
initFolder("normal")
|
||||
MainMenu()
|
||||
|
||||
# noticed an issue where this doesn't run standalone anymore. maybe this will fix it. idk
|
||||
if len(arglist) == 0:
|
||||
# log.debug("no args were passed!")
|
||||
initFolder("noarg")
|
||||
MainMenu()
|
||||
# 09/28/22- ok this will semi work for now. if any arg other than the ones defined above are passed it'll ignore them and run like none were passed
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user