feat: automated tts scrapper

Author: tomaspalma

services/ni/scrapper/01_secrets.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: tts-scrapper-secrets
+  namespace: tts
+type: Opaque
+stringData:
+  TTS_SCRAPY_USER: <FILL-IN>
+  TTS_SCRAPY_PASSWORD: <FILL-IN>
+  TTS_SCRAPY_YEAR: <FILL-IN>
+  POSTGRES_DB: <FILL-IN>
+  POSTGRES_HOST: <FILL-IN>
+  POSTGRES_PASSWORD: <FILL-IN>
+  POSTGRES_PORT: <FILL-IN>
+  POSTGRES_USER: <FILL-IN>

services/ni/scrapper/02_secrets-staging.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: tts-scrapper-secrets
+  namespace: tts-staging
+type: Opaque
+stringData:
+  TTS_SCRAPY_USER: <FILL-IN>
+  TTS_SCRAPY_PASSWORD: <FILL-IN>
+  TTS_SCRAPY_YEAR: <FILL-IN>
+  POSTGRES_DB: <FILL-IN>
+  POSTGRES_HOST: <FILL-IN>
+  POSTGRES_PASSWORD: <FILL-IN>
+  POSTGRES_PORT: <FILL-IN>
+  POSTGRES_USER: <FILL-IN>

services/ni/scrapper/03_cron.yaml

@@ -0,0 +1,38 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: automated-scrapper
+  namespace: tts
+  annotations:
+    keel.sh/policy: "force"
+    keel.sh/match-tag: "true" 
+spec:
+  schedule: "0 */6 * * *"   # every 3 hours
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          annotations:
+            keel.sh/policy: force
+        spec:
+          containers:
+          - name: runner
+            image: registry.niaefeup.pt/niaefeup/uporto-schedule-scrapper:master
+            command: ["/bin/sh", "-c"]
+            args:
+              - |
+                apt update && apt install -y postgresql-client;
+                psql;
+                make clean;
+                make info faculties courses course_units course_metadata;
+                make dump;
+                make convert_postgres;
+                PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p 5432 -U $POSTGRES_USER -d $POSTGRES_DB -f ./scripts/dump/data/01_data.sql;  
+            envFrom:
+            - secretRef:
+                name: tts-scrapper-secrets
+            imagePullPolicy: Always
+          restartPolicy: OnFailure
+          imagePullSecrets:
+            - name: harbor-pull-secret
+

services/ni/scrapper/04_cron-staging.yaml

@@ -0,0 +1,38 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: automated-scrapper
+  namespace: tts-staging
+  annotations:
+    keel.sh/policy: "force"
+    keel.sh/match-tag: "true" 
+spec:
+  schedule: "* * 4 * *" 
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          annotations:
+            keel.sh/policy: force
+        spec:
+          containers:
+          - name: runner
+            image: registry.niaefeup.pt/niaefeup/uporto-schedule-scrapper:master
+            command: ["/bin/sh", "-c"]
+            args:
+              - |
+                apt update && apt install -y postgresql-client;
+                psql;
+                make clean;
+                make info faculties courses course_units course_metadata slots;
+                make dump;
+                make convert_postgres;
+                PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p 5432 -U $POSTGRES_USER -d $POSTGRES_DB -f ./scripts/dump/data/01_data.sql;  
+            envFrom:
+            - secretRef:
+                name: tts-scrapper-secrets
+            imagePullPolicy: Always
+          restartPolicy: OnFailure
+          imagePullSecrets:
+            - name: harbor-pull-secret
+

services/ni/scrapper/05_vacancies-cron.yaml

@@ -0,0 +1,37 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: class-vacancies-scrapper
+  namespace: tts
+  annotations:
+    keel.sh/policy: "force"
+    keel.sh/match-tag: "true"
+spec:
+  schedule: "0 */1 * * *"   # every 3 hours
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          annotations:
+            keel.sh/policy: force
+        spec:
+          containers:
+          - name: runner
+            image: registry.niaefeup.pt/niaefeup/uporto-schedule-scrapper:master
+            command: ["/bin/sh", "-c"]
+            args:
+              - |
+                apt update && apt install -y postgresql-client;
+                psql;
+                make clean;
+                make class_vacancies;
+                make dump;
+                make convert_postgres;
+                PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p 5432 -U $POSTGRES_USER -d $POSTGRES_DB -f ./scripts/dump/data/01_data.sql;
+            envFrom:
+            - secretRef:
+                name: tts-scrapper-secrets
+            imagePullPolicy: Always
+          restartPolicy: OnFailure
+          imagePullSecrets:
+            - name: harbor-pull-secret