Wiki: Kubernetes/Utilities / File Location Listing

Although Kubernetes is not required, the project was written with a k8s deployment in mind.

The built image has a custom entry point allowing a container to be launched in one of several modes

• crawl
• reval
• reindex
• portal

The mode can be changed by setting environment variable MODE

---

Manifest

The following manifest creates a deployment, Loadbalancer and crons for crawls and validations, using a common NFS mount as the storage volume

---

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: searchportal
  name: searchportal
spec:
  selector:
    matchLabels:
      app: searchportal
  template:
    metadata:
      labels:
        app: searchportal
    spec:
      containers:
        - name: searchportal
          image: bentasker12/file_location_listing:0.1
          imagePullPolicy: IfNotPresent
          env:
          - name: DB_PATH
            value: "/search_db"
          - name: CONFIG_BASE
            value: "/search_db/"
          - name: NUM_THREADS
            value: "4"
          ports:
            - containerPort: 5000
              name: http-searchport
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /
              port: 5000
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 30
            successThreshold: 1
            timeoutSeconds: 2
          livenessProbe:
            failureThreshold: 3
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            tcpSocket:
              port: 5000
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 250m
              memory: 750Mi
          volumeMounts:
            - mountPath: /search_db
              name: searchstore
      volumes:
        - name: searchstore
          nfs:
            server: 192.168.3.9
            path: /volume1/searchfiles
            readOnly: false

---

apiVersion: v1
kind: Service
metadata:
  name: searchportal
spec:
  ports:
    - port: 5000
      protocol: TCP
      targetPort: http-searchport
  selector:
    app: searchportal
  sessionAffinity: None
  type: LoadBalancer


---

apiVersion: batch/v1
kind: CronJob
metadata:
  name: filelisting-spider
spec:
  schedule: "0 4 * * *"
  failedJobsHistoryLimit: 5
  successfulJobsHistoryLimit: 5
  jobTemplate:
    spec:
      template:
        spec:
            restartPolicy: OnFailure
            volumes:
            - name: searchstore
              nfs:
                server: 192.168.3.9
                path: /volume1/searchfiles
                readOnly: false

            hostname: search-crawler
            containers:
            - name: filelisting-spider
              image: bentasker12/file_location_listing:0.1
              imagePullPolicy: IfNotPresent
              volumeMounts:
                    - mountPath: /search_db
                      name: searchstore
                      readOnly: false
              env:
                - name: MODE
                  value: "crawl"
                - name: DB_PATH
                  value: "/search_db"
                - name: CONFIG_BASE
                  value: "/search_db/"

---

apiVersion: batch/v1
kind: CronJob
metadata:
  name: filelisting-revalidate
spec:
  schedule: "0 22 * * *"
  failedJobsHistoryLimit: 5
  successfulJobsHistoryLimit: 5
  jobTemplate:
    spec:
      template:
        spec:
            restartPolicy: OnFailure
            volumes:
            - name: searchstore
              nfs:
                server: 192.168.3.9
                path: /volume1/searchfiles
                readOnly: false

            hostname: filelisting-reval
            containers:
            - name: filelisting-reval
              image: bentasker12/file_location_listing:0.1
              imagePullPolicy: IfNotPresent
              volumeMounts:
                    - mountPath: /search_db
                      name: searchstore
                      readOnly: false
              env:
                - name: MODE
                  value: "reval"                  
                - name: REVAL_COUNT
                  value: "1000"
                - name: DB_PATH
                  value: "/search_db"
                - name: CONFIG_BASE
                  value: "/search_db/"