first commit

scripts/ExportRequest_Worker.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+$basePath = realpath(__DIR__ . '/..');
+require_once $basePath . '/classes/Process_ExportRequest.php';
+
+//require_once "../classes/Process_ExportRequest.php";
+
+(new Process_ExportRequest())->process();
+?>

scripts/FWCCronScript.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# -- Development -- 
+cd /var/www/html/clients/dec-international/realestatedatainc
+# -- Production -- 
+#cd /var/www/html/webapps/dec-international/realestatedatainc
+php ./scripts/ExportRequest_Worker.php
+

scripts/Import_VoterHistory.php

@@ -0,0 +1,6 @@
+<?php
+
+require_once "../classes/Import_VoterHistory.php";
+
+(new Import_VoterHistory())->process();
+?>

scripts/Import_VoterRecords.php

@@ -0,0 +1,6 @@
+<?php
+
+require_once "../classes/Import_VoterRecords.php";
+
+(new Import_VoterRecords())->process();
+?>

scripts/fetch_and_process_fwc_public_records.sh

@@ -0,0 +1,207 @@
+#!/bin/bash
+
+# ===========================================================================================
+# FWC Public Records Download + Import Script
+# Downloads FWC public record files and imports recreational and commercial licenses
+# Optional: FWC_DB_HOST, FWC_DB_NAME, FWC_DB_USER, and FWC_DB_PASS override xml/settings.xml.
+# Optional: FWC_WEBHOOK_URL enables Discord notifications.
+# ===========================================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+BASE_DIR="${PROJECT_DIR}/data/fwcdata"
+PAGE_URL="https://myfwc.com/license/public-record/"
+
+CURRENT_DIR="${BASE_DIR}/licenses/current"
+ARCHIVE_DIR="${BASE_DIR}/licenses/archive"
+LOG_DIR="${BASE_DIR}/licenses/logs"
+
+RECREATIONAL_IMPORTER="${PROJECT_DIR}/classes/imports/Import_RecreationalLicenses.php"
+COMMERCIAL_IMPORTER="${PROJECT_DIR}/classes/imports/Import_CommercialLicenses.php"
+
+LOG_FILE="${LOG_DIR}/fwc_fetch_$(date +%Y%m%d).log"
+
+mkdir -p "${CURRENT_DIR}/recreational" \
+         "${CURRENT_DIR}/commercial" \
+         "${ARCHIVE_DIR}/recreational" \
+         "${ARCHIVE_DIR}/commercial" \
+         "${LOG_DIR}"
+
+logMessage() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
+}
+
+downloadFile() {
+
+    local FILE_URL="$1"
+    local DATASET_TYPE="$2"
+
+    local FILE_NAME
+    local CURRENT_FILE
+    local TEMP_FILE
+    local ARCHIVE_FILE
+
+    FILE_NAME=$(basename "${FILE_URL}")
+    CURRENT_FILE="${CURRENT_DIR}/${DATASET_TYPE}/${FILE_NAME}"
+    TEMP_FILE="${CURRENT_FILE}.tmp"
+    ARCHIVE_FILE="${ARCHIVE_DIR}/${DATASET_TYPE}/${FILE_NAME}.$(date +%Y%m%d_%H%M%S)"
+
+    logMessage "Downloading ${DATASET_TYPE}: ${FILE_NAME}"
+
+    curl -L -f -s -S "${FILE_URL}" -o "${TEMP_FILE}" >> "${LOG_FILE}" 2>&1
+
+    if [ $? -ne 0 ]; then
+        logMessage "ERROR: Failed to download ${FILE_NAME}"
+        rm -f "${TEMP_FILE}"
+        return 1
+    fi
+
+    if [ ! -s "${TEMP_FILE}" ]; then
+        logMessage "ERROR: Downloaded file is empty: ${FILE_NAME}"
+        rm -f "${TEMP_FILE}"
+        return 1
+    fi
+
+    if [ -f "${CURRENT_FILE}" ] && cmp -s "${CURRENT_FILE}" "${TEMP_FILE}"; then
+        logMessage "No changes detected for ${FILE_NAME}"
+        rm -f "${TEMP_FILE}"
+        return 0
+    fi
+
+    if [ -f "${CURRENT_FILE}" ]; then
+        cp -p "${CURRENT_FILE}" "${ARCHIVE_FILE}"
+        logMessage "Archived old file to ${ARCHIVE_FILE}"
+    fi
+
+    mv "${TEMP_FILE}" "${CURRENT_FILE}"
+    logMessage "Updated current file: ${CURRENT_FILE}"
+    return 0
+}
+
+logMessage "FWC public record fetch started."
+
+TMP_PAGE=$(mktemp)
+
+curl -L -f -s -S "${PAGE_URL}" -o "${TMP_PAGE}" >> "${LOG_FILE}" 2>&1
+
+if [ $? -ne 0 ]; then
+    logMessage "ERROR: Could not fetch public record page."
+    rm -f "${TMP_PAGE}"
+    exit 1
+fi
+
+# Pull all TXT links from the page.
+URLS=$(grep -oiE 'href="[^"]+\.txt"' "${TMP_PAGE}" \
+    | cut -d'"' -f2 \
+    | grep '/media/licenses/' \
+    | grep -vi 'fileinfo' \
+    | sort -u)
+
+rm -f "${TMP_PAGE}"
+
+if [ -z "${URLS}" ]; then
+    logMessage "ERROR: No license text files found."
+    exit 1
+fi
+
+DOWNLOAD_ERRORS=0
+RECREATIONAL_DOWNLOADS=0
+COMMERCIAL_DOWNLOADS=0
+SKIPPED_FILES=0
+
+while read -r LINK; do
+
+    if [[ "${LINK}" == http* ]]; then
+        FILE_URL="${LINK}"
+    else
+        FILE_URL="https://myfwc.com${LINK}"
+    fi
+
+    FILE_NAME=$(basename "${FILE_URL}" | tr '[:upper:]' '[:lower:]')
+
+    case "${FILE_NAME}" in
+
+        recallcurrent*.txt)
+            downloadFile "${FILE_URL}" "recreational" || DOWNLOAD_ERRORS=1
+            RECREATIONAL_DOWNLOADS=$((RECREATIONAL_DOWNLOADS + 1))
+            ;;
+
+        *.txt)
+            downloadFile "${FILE_URL}" "commercial" || DOWNLOAD_ERRORS=1
+            COMMERCIAL_DOWNLOADS=$((COMMERCIAL_DOWNLOADS + 1))
+            ;;
+
+        *)
+            logMessage "Skipping unrecognized file: ${FILE_NAME}"
+            SKIPPED_FILES=$((SKIPPED_FILES + 1))
+            ;;
+
+    esac
+
+done <<< "${URLS}"
+
+logMessage "Discovered ${RECREATIONAL_DOWNLOADS} recreational files, ${COMMERCIAL_DOWNLOADS} commercial files, and skipped ${SKIPPED_FILES} files."
+
+if [ "${DOWNLOAD_ERRORS}" -ne 0 ]; then
+    logMessage "ERROR: One or more files failed to download."
+    exit 1
+fi
+
+# ==================
+# Run PHP Importers
+# ==================
+
+if [ -f "${RECREATIONAL_IMPORTER}" ]; then
+
+    logMessage "Starting recreational PHP import."
+
+    php "${RECREATIONAL_IMPORTER}" \
+        --base-dir="${BASE_DIR}" \
+        --recreational-dir="${CURRENT_DIR}/recreational" \
+        >> "${LOG_FILE}" 2>&1
+
+    if [ $? -eq 0 ]; then
+        logMessage "Recreational PHP import completed successfully."
+    else
+        logMessage "ERROR: Recreational PHP import failed."
+        exit 1
+    fi
+
+else
+    logMessage "ERROR: Recreational PHP importer not found: ${RECREATIONAL_IMPORTER}"
+    exit 1
+fi
+
+if [ -f "${COMMERCIAL_IMPORTER}" ]; then
+
+    logMessage "Starting commercial PHP import."
+
+    php "${COMMERCIAL_IMPORTER}" \
+        --base-dir="${BASE_DIR}" \
+        --commercial-dir="${CURRENT_DIR}/commercial" \
+        >> "${LOG_FILE}" 2>&1
+
+    if [ $? -eq 0 ]; then
+        logMessage "Commercial PHP import completed successfully."
+    else
+        logMessage "ERROR: Commercial PHP import failed."
+        exit 1
+    fi
+
+else
+    logMessage "ERROR: Commercial PHP importer not found: ${COMMERCIAL_IMPORTER}"
+    exit 1
+fi
+
+# ==========================
+# Delete Old Archived Files
+# ==========================
+
+find "${ARCHIVE_DIR}" -type f -mtime +60 -delete
+logMessage "Deleted archive files older than 60 days."
+
+find "${LOG_DIR}" -type f -name "*.log" -mtime +60 -delete
+logMessage "Deleted log files older than 60 days."
+
+logMessage "FWC public record fetch complete."