From f72ed312c93f4177318f8eeee3282df795ae66d1 Mon Sep 17 00:00:00 2001
From: iRaven <44326347+iRaven4522@users.noreply.github.com>
Date: Wed, 7 Sep 2022 02:10:57 -0500
Subject: [PATCH] semi initial commit

---
 .gitignore          |    3 +
 main.py             |  118 +++++
 valid_feed_list.txt | 1166 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 1287 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 main.py
 create mode 100644 valid_feed_list.txt

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..81a71a4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+CCTVCrawl*
+devscrape*
+*.log*
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..185b25f
--- /dev/null
+++ b/main.py
@@ -0,0 +1,118 @@
+import logging
+import os
+import wget
+import time
+import datetime
+import requests
+import csv
+import threading
+
+# TranstarArchive Main PY
+# author: iRaven (https://iravenhome.net)
+# Started 9/6/22.
+
+#version = "0.1"
+
+# Configure logging
+logging.basicConfig(filename="TransArchive.log",
+                    format='%(asctime)s %(message)s',
+                    filemode='a')
+log = logging.getLogger()
+log.setLevel(logging.DEBUG)
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+#define local vars
+
+#time
+initcurtime = time.localtime()
+# currtime = (f'{initcurtime.tm_hour}-{initcurtime.tm_min}-{initcurtime.tm_sec}')
+curtime = time.strftime("%H-%M-%S", initcurtime)
+curdate = (f'{initcurtime.tm_year}-{initcurtime.tm_mon}-{initcurtime.tm_mday}')
+
+csvfile = (f"TransArchive_{curdate}.csv") # mainly for debugging but to show status codes of requests for feeds
+csvfields = ['time','camera_id', 'status'] # ^
+
+# very important var: where we get everything
+cctvurl = (f'https://www.houstontranstar.org/snapshots/cctv/') # Last updated Sep-2022
+# very important var: list of valid camera feeds generated by devs (likely included and updated in repo)
+vftxt = open("valid_feed_list.txt", "r")
+vfdata = vftxt.read()
+vfeeds = vfdata.split("\n")
+vftxt.close()
+# TODO: very important (future) var: custom paths
+# TODO: archfolder = "archive/"
+
+
+######## Core functions
+# initialize folder
+def initFolder():
+    log.info(f'TranstarArchive was started')
+    if not os.path.exists("archive/"):
+        log.info("archive folder doesn't exist. creating it")
+        os.makedirs("archive/")
+    elif not os.path.exists(f"archive/{curdate}/"):
+        log.warning("archive folder for todays date doesn't exist (expected LOL), creating it")
+        os.makedirs(f"archive/{curdate}")
+    elif not os.path.exists(f"TransstarArchive_{curdate}.csv"):
+        log.error("csv file doesn't exist. creating it for today")
+        with open(csvfile, 'a') as listfile:
+            csvwrite = csv.writer(listfile)
+            csvwrite.writerow(csvfields)
+    else:
+        log.info("archive download folder exists. we're jud")
+
+# image downloader
+def imageDownload(url):
+    validcount=0
+    invalidfeeds=[]
+    log.info(f"Beginning to archive images at {curtime}")
+    for feed in vfeeds:
+        rq = requests.get(f'{url}{feed}.jpg')
+        if rq.headers['content-type'] == "image/jpeg":
+            validcount = validcount + 1
+            csvvalid = [curtime,feed,'Success']
+            print(f'Valid camera feed is being archived: {feed}\n')
+            log.info(f'Valid camera feed is being archived: {feed}')
+            wget.download(rq.url,f'archive/{curdate}/{feed}_{curtime}.jpg')
+            with open(csvfile, 'a') as listfile:
+                csvwrite = csv.writer(listfile)
+                csvwrite.writerow(csvvalid)
+        else:
+            log.error(f"Invalid feed was not archived: {feed}")
+            invalidfeeds.append(feed)
+        
+    log.info(f'Archive at {curtime} finished successfully, {validcount} feeds were archived in this cycle')
+    # with open("invalid_feeds_log.txt", 'a') as invlog:
+    #     for vals in invlog:
+    #         invlog.write(f'invalid feeds found at archive {curdate} at {curtime}:\n{vals}\n#### END END END ####\n either transstar is having issues or has reconfigured feed numbers, open an issue in the repo: https://github.com/iraven4522/TranstarArchive\n')
+
+# loops the download function every like 3 or so mins
+def imgDownloadLoop(secs):
+    log.info(f'delay loop was called for {secs}, starting now')
+    threading.Timer(f'{secs}.0', imgDownloadLoop).start()
+
+# main menu
+def MainMenu():
+    log.info(f'main menu started')
+    print("Welcome to TranstarArchive, an application which lets you archive and keep store of Houston Transtar traffic camera feeds")
+    print("Normally HT doesn't archive these via video, however this app lets you have a locally stored archive")
+    print("Developed by iRaven. https://iravenhome.net / https://github.com/iRaven4522 (ponies are cyoot :3)")
+    menuin = input("What do you want to do? archive/exit ")
+    if menuin == "archive":
+        log.info("User chose archive")
+        print("Start Archive")
+        print("This will start archiving continuously until the program is closed with Ctrl+C or closing it")
+        print(f'You\'re about to run an automated image download on a website that may take a few minutes, depending on server reliability and bandwidth')
+        menuin1 = input("If you want to continue please type the phrase \"startrans\" (without quotes) or type no: ")
+        if menuin1 == "startrans":
+            imageDownload(cctvurl)
+            log.info("returned to menu for delay")
+            imgDownloadLoop(195)
+        else:
+            exit()
+    else:
+        exit()
+
+# Run the shit (finally)
+initFolder() # check folder and csv file first (send log start message)
+MainMenu()
diff --git a/valid_feed_list.txt b/valid_feed_list.txt
new file mode 100644
index 0000000..637e370
--- /dev/null
+++ b/valid_feed_list.txt
@@ -0,0 +1,1166 @@
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+301
+303
+304
+305
+306
+307
+308
+309
+310
+311
+312
+313
+314
+315
+316
+317
+318
+319
+320
+321
+322
+323
+324
+325
+326
+327
+328
+329
+330
+331
+332
+333
+334
+335
+336
+337
+338
+339
+340
+341
+342
+343
+344
+345
+346
+347
+348
+349
+350
+351
+352
+353
+355
+356
+357
+401
+402
+403
+404
+405
+406
+407
+408
+409
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+430
+498
+499
+501
+502
+503
+504
+505
+506
+507
+508
+509
+510
+511
+512
+513
+514
+515
+516
+517
+518
+519
+520
+521
+522
+523
+524
+525
+526
+527
+528
+529
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+540
+541
+542
+543
+544
+601
+602
+603
+604
+605
+606
+607
+608
+609
+610
+611
+612
+613
+614
+615
+616
+617
+618
+619
+620
+621
+622
+623
+624
+625
+626
+627
+628
+629
+630
+631
+632
+633
+634
+635
+636
+637
+638
+639
+640
+641
+642
+643
+644
+645
+646
+647
+648
+649
+650
+651
+652
+653
+654
+655
+656
+702
+703
+704
+705
+706
+707
+708
+709
+710
+711
+712
+713
+714
+715
+716
+717
+718
+719
+720
+802
+804
+805
+807
+808
+809
+810
+811
+812
+813
+814
+815
+816
+817
+819
+820
+821
+822
+823
+824
+825
+826
+827
+828
+829
+830
+901
+902
+903
+904
+905
+906
+907
+908
+909
+910
+911
+912
+913
+914
+915
+916
+917
+918
+919
+920
+921
+922
+923
+924
+925
+926
+927
+928
+929
+930
+931
+932
+933
+934
+935
+1001
+1002
+1003
+1004
+1005
+1006
+1007
+1008
+1009
+1010
+1011
+1012
+1013
+1014
+1015
+1016
+1017
+1018
+1019
+1020
+1021
+1022
+1023
+1024
+1025
+1026
+1027
+1028
+1029
+1030
+1031
+1032
+1033
+1034
+1035
+1036
+1102
+1103
+1104
+1105
+1106
+1107
+1108
+1109
+1110
+1111
+1112
+1113
+1114
+1115
+1117
+1118
+1119
+1120
+1121
+1122
+1123
+1124
+1125
+1126
+1127
+1128
+1129
+1130
+1131
+1132
+1133
+1134
+1135
+1136
+1137
+1138
+1139
+1140
+1141
+1142
+1143
+1144
+1145
+1146
+1147
+1148
+1149
+1150
+1152
+1153
+1154
+1155
+1156
+1157
+1158
+1159
+1160
+1161
+1162
+1163
+1164
+1165
+1166
+1167
+1168
+1169
+1170
+1171
+1172
+1173
+1174
+1175
+1176
+1177
+1178
+1179
+1180
+1181
+1182
+1183
+1184
+1185
+1186
+1187
+1188
+1189
+1190
+1191
+1192
+1193
+1194
+1195
+1202
+1203
+1204
+1205
+1206
+1207
+1208
+1209
+1210
+1211
+1212
+1213
+1214
+1215
+1216
+1217
+1218
+1219
+1220
+1221
+1222
+1223
+1224
+1225
+1226
+1227
+1228
+1229
+1230
+1231
+1232
+1233
+1234
+1235
+1236
+1237
+1238
+1239
+1240
+1241
+1301
+1302
+1303
+1304
+1306
+1307
+1308
+1310
+1311
+1312
+1313
+1314
+1315
+1316
+1401
+1402
+1403
+1404
+1405
+1406
+1407
+1408
+1409
+1410
+1411
+1412
+1413
+1414
+1415
+1416
+1417
+1418
+1419
+1420
+1421
+1422
+1423
+1424
+1425
+1426
+1427
+1428
+1429
+1430
+1431
+1432
+1433
+1434
+1435
+1436
+1437
+1438
+1439
+1440
+1441
+1442
+1443
+1444
+1445
+1446
+1447
+1448
+1449
+1450
+1451
+1452
+1453
+1454
+1455
+1456
+1457
+1458
+1459
+1460
+1461
+1462
+1463
+1464
+1465
+1466
+1467
+1468
+1469
+1470
+1501
+1502
+1503
+1504
+1505
+1506
+1507
+1508
+1601
+1602
+1603
+1604
+1605
+1801
+1802
+1803
+1804
+1805
+1806
+1807
+1808
+1809
+1810
+1811
+1812
+1813
+1814
+1815
+1816
+1905
+1906
+1907
+1908
+1909
+1910
+1911
+1912
+1913
+1914
+1915
+1916
+1917
+1918
+1919
+1920
+1921
+1923
+1924
+1925
+1926
+1927
+1928
+1929
+2101
+2102
+2103
+2104
+2105
+2106
+2107
+2108
+2109
+2110
+2111
+2112
+2113
+2114
+2115
+2116
+2117
+2118
+2119
+2120
+2121
+2122
+2123
+2124
+2125
+2126
+2127
+2128
+2129
+2130
+2131
+2132
+2133
+2134
+2135
+2136
+2137
+2201
+2206
+2209
+2212
+2217
+2226
+2229
+2232
+2236
+2241
+2245
+2248
+2250
+2251
+2252
+2253
+2256
+2261
+2262
+2263
+2264
+2265
+2266
+2267
+2268
+2269
+2270
+2271
+2272
+2273
+2274
+2275
+2276
+2277
+2278
+2279
+2280
+2281
+2282
+2283
+2301
+2302
+2303
+2304
+2305
+2306
+2307
+2308
+2309
+2310
+2311
+2312
+2313
+2314
+2315
+2316
+2317
+2318
+2319
+2320
+2321
+2322
+2323
+2324
+2325
+2326
+2327
+2328
+2329
+2330
+2331
+2332
+2333
+2334
+2335
+2336
+2337
+2338
+2339
+2340
+2341
+2342
+2343
+2344
+2345
+2346
+2347
+2348
+2349
+2350
+2351
+2352
+2353
+2354
+2355
+2356
+2357
+2358
+2359
+2360
+2361
+2362
+2363
+2364
+2365
+2366
+2367
+2368
+2369
+2370
+2371
+2372
+2373
+2374
+2375
+2376
+2377
+2378
+2380
+2381
+2382
+2383
+2384
+2385
+2386
+2387
+2388
+2389
+2390
+2391
+2392
+2393
+2394
+2395
+2396
+2397
+2398
+2399
+2401
+2402
+2403
+2404
+2405
+2406
+2407
+2408
+2409
+2410
+2411
+2412
+2413
+2414
+2415
+2416
+2417
+2418
+2419
+2420
+2421
+2422
+2423
+2424
+2425
+2426
+2427
+2428
+2429
+2430
+2431
+2432
+2433
+2434
+2435
+2436
+2437
+2438
+2439
+2440
+2441
+2950
+3001
+3021
+3022
+3031
+3032
+3081
+3101
+3102
+3103
+3104
+3105
+3106
+3107
+3202
+3203
+3204
+3205
+3206
+3207
+3208
+3209
+3210
+3211
+3212
+3218
+3219
+3220
+3221
+3222
+3223
+3224
+3225
+3226
+3227
+3302
+3303
+3304
+3305
+3306
+3307
+3308
+3309
+3310
+3311
+3312
+3313
+3314
+3315
+3316
+3318
+3319
+3320
+3321
+3322
+3323
+3324
+3403
+3404
+3405
+3406
+3501
+3502
+3503
+3504
+3505
+3506
+3507
+3508
+3509
+3510
+3511
+3512
+3513
+3514
+3515
+3516
+3517
+3518
+3519
+3520
+3521
+3522
+3523
+3524
+3526
+3527
+3528
+3529
+3530
+3600
+3601
+3602
+3603
+3604
+3605
+3606
+3607
+3608
+3609
+3801
+3802
+3803
+4000
+4002
+4003
+4004
+4005
+4006
+4007
+4008
+4009
+4010
+4011
+4012
+4013
+4014
+4015
+4016
+4017
+4018
+4019
+4020
+4021
+4022
+4023
+4025
+4026
+4027
+4028
+4029
+4030
+4031
+4032
+4033
+4034
+4035
+4036
+4037
+4038
+4039
+4101
+4102
+4103
+4104
+4105
+4106
+4107
+4108
+4109
+4110
+4111
+4112
+4113
+4114
+4115
+4116
+4117
+4118
+4119
+4120
+4121
+4122
+4123
+4124
+4125
+4126
+4127
+4128
+4129
+4130
+4131
+4132
+4133
+4150
+4170
+4180
+4190
+4201
+4202
+4203
+4204
+4205
+4206
+4207
+4208
+4209
+4210
+4211
+4212
+4213
+4214
+4215
+4216
+4217
+4240
+4241
+4250
+4251
+4260
+4270
+4271
+4280
+4290
+4440
+4441
+4450
+4451
+4460
+4470
+4471
+4480
+4490
+5200
+5201
+5203
+6140