jric11/atlhousingreport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Import for Large CSV

Browse Source
This commit is contained in:
James Richie
2026-05-21 19:16:27 -04:00
parent 3c5401f82b
commit 8a3cb15e55
22 changed files with 1571 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
classes/Consolidation.php
+67 -149
View File
@@ -2,12 +2,12 @@
class Consolidation extends Base {
// Variables.
// Variables
private $current_user_serial = 0;
private $current_user_id = "";
private $current_user_name = "";
private $current_user_email = "";
private $current_user_id = "";
private $current_user_name = "";
private $current_user_email = "";
// =================
// Class Constructor
@@ -20,9 +20,9 @@ class Consolidation extends Base {
date_default_timezone_set(self::TIMEZONE);
$this->current_user_serial = $_SESSION[self::USER_SERIAL] ?? 0;
$this->current_user_id = $_SESSION[self::USER_ID] ?? "";
$this->current_user_name = $_SESSION[self::USER_NAME] ?? "";
$this->current_user_email = $_SESSION[self::USER_EMAIL] ?? "";
$this->current_user_id = $_SESSION[self::USER_ID] ?? "";
$this->current_user_name = $_SESSION[self::USER_NAME] ?? "";
$this->current_user_email = $_SESSION[self::USER_EMAIL] ?? "";
}
// ======================
@@ -47,6 +47,11 @@ class Consolidation extends Base {
case "run":
$XSLParms = array();
$permit_record_count = 0;
$processed_record_count = 0;
$connection = null;
$sql_insert = "insert into rdiconsolidations
( rdiconsolidation_permit_facts_id,
@@ -85,150 +90,73 @@ class Consolidation extends Base {
rdiconsolidation_ytd_value,
rdiconsolidation_12mth_prev_permits,
rdiconsolidation_date )
values ( :rdiconsolidation_permit_facts_id,
:rdiconsolidation_entry_date_str,
:rdiconsolidation_entry_date,
:rdiconsolidation_permit_number,
:rdiconsolidation_permit_date_str,
:rdiconsolidation_permit_date,
:rdiconsolidation_project_addr,
:rdiconsolidation_sub_div_name,
:rdiconsolidation_project_type,
:rdiconsolidation_company,
:rdiconsolidation_address,
:rdiconsolidation_city,
:rdiconsolidation_state,
:rdiconsolidation_zip,
:rdiconsolidation_phone,
:rdiconsolidation_project_city,
:rdiconsolidation_project_state,
:rdiconsolidation_size,
:rdiconsolidation_units,
:rdiconsolidation_building_type,
:rdiconsolidation_value,
:rdiconsolidation_work_type,
:rdiconsolidation_county,
:rdiconsolidation_lot,
:rdiconsolidation_dist_ll,
:rdiconsolidation_ct1,
:rdiconsolidation_owner_name,
:rdiconsolidation_owner_address,
:rdiconsolidation_owner_city,
:rdiconsolidation_owner_state,
:rdiconsolidation_owner_zip,
:rdiconsolidation_owner_phone,
:rdiconsolidation_ytd_permits,
:rdiconsolidation_ytd_value,
:rdiconsolidation_12mth_prev_permits,
:rdiconsolidation_date ) ";
$connection = $this->connect();
select coalesce(permit_permit_facts_id, 0),
coalesce(permit_entry_date_str, ''),
coalesce(permit_entry_date, '0000-00-00'),
coalesce(permit_permit_number, ''),
coalesce(permit_permit_date_str, ''),
coalesce(permit_permit_date, '0000-00-00'),
coalesce(permit_project_addr, ''),
coalesce(permit_sub_div_name, ''),
coalesce(permit_project_type, ''),
coalesce(permit_company, ''),
coalesce(permit_address, ''),
coalesce(permit_city, ''),
coalesce(permit_state, ''),
coalesce(permit_zip, ''),
coalesce(permit_phone, ''),
coalesce(permit_project_city, ''),
coalesce(permit_project_state, ''),
coalesce(permit_size, 0),
coalesce(permit_units, 0),
coalesce(permit_building_type, 0),
coalesce(permit_value, 0),
coalesce(permit_work_type, 0),
coalesce(permit_county, ''),
coalesce(permit_lot, ''),
coalesce(permit_dist_ll, ''),
coalesce(permit_ct1, 0),
coalesce(permit_owner_name, ''),
coalesce(permit_owner_address, ''),
coalesce(permit_owner_city, ''),
coalesce(permit_owner_state, ''),
coalesce(permit_owner_zip, ''),
coalesce(permit_owner_phone, ''),
coalesce(permit_ytd_permits, 0),
coalesce(permit_ytd_value, 0),
coalesce(permit_12mth_prev_permits, 0),
current_date()
from permits";
$connection->beginTransaction();
$statement = $connection->prepare($sql_insert);
try {
$rdi_records = $this->getRdiTable();
$rdi_record_count = $rdi_records->count();
$connection = $this->connect();
$processed_record_count = 0;
$connection->beginTransaction();
foreach ($rdi_records as $record) {
$permit_record_count = (integer) $connection->query("select count(*) from permits")->fetchColumn();
$processed_record_count = (integer) $connection->exec($sql_insert);
$rdiconsolidation_permit_facts_id = (integer) $record->permit_facts_id ?: 0;
$rdiconsolidation_entry_date_str = (integer) $record->entry_date_str ?: "";
$rdiconsolidation_entry_date = (string) $record->entry_date ?: "0000-00-00";
$rdiconsolidation_permit_number = (string) $record->permit_number ?: "";
$rdiconsolidation_permit_date_str = (string) $record->permit_date_str ?: "";
$rdiconsolidation_permit_date = (string) $record->permit_date ?: "0000-00-00";
$rdiconsolidation_project_addr = (string) $record->project_addr ?: "";
$rdiconsolidation_sub_div_name = (string) $record->sub_div_name ?: "";
$rdiconsolidation_project_type = (string) $record->project_type ?: "";
$rdiconsolidation_company = (string) $record->company ?: "";
$rdiconsolidation_address = (string) $record->address ?: "";
$rdiconsolidation_city = (string) $record->city ?: "";
$rdiconsolidation_state = (string) $record->state ?: "";
$rdiconsolidation_zip = (string) $record->zip ?: "";
$rdiconsolidation_phone = (string) $record->phone ?: "";
$rdiconsolidation_project_city = (string) $record->project_city ?: "";
$rdiconsolidation_project_state = (string) $record->project_state ?: "";
$rdiconsolidation_size = (integer) $record->size ?: 0;
$rdiconsolidation_units = (integer) $record->units ?: 0;
$rdiconsolidation_building_type = (integer) $record->building_type ?: 0;
$rdiconsolidation_value = (integer) $record->value ?: 0;
$rdiconsolidation_work_type = (integer) $record->work_type ?: 0;
$rdiconsolidation_county = (string) $record->county ?: "";
$rdiconsolidation_lot = (string) $record->lot ?: "";
$rdiconsolidation_dist_ll = (string) $record->dist_ll ?: "";
$rdiconsolidation_ct1 = (integer) $record->ct1 ?: 0;
$rdiconsolidation_owner_name = (string) $record->owner_name ?: "";
$rdiconsolidation_owner_address = (string) $record->owner_address ?: "";
$rdiconsolidation_owner_city = (string) $record->owner_city ?: "";
$rdiconsolidation_owner_state = (string) $record->owner_state ?: "";
$rdiconsolidation_owner_zip = (string) $record->owner_zip ?: "";
$rdiconsolidation_owner_phone = (string) $record->owner_phone ?: "";
$rdiconsolidation_ytd_permits = (integer) $record->ytd_permits ?: 0;
$rdiconsolidation_ytd_value = (integer) $record->ytd_value ?: 0;
$rdiconsolidation_12mth_prev_permits = (integer) $record->{"12mth_prev_permits"} ?: 0;
$rdiconsolidation_date = date("Y-m-d");
$statement->bindValue(":rdiconsolidation_permit_facts_id", $rdiconsolidation_permit_facts_id);
$statement->bindValue(":rdiconsolidation_entry_date_str", $rdiconsolidation_entry_date_str);
$statement->bindValue(":rdiconsolidation_entry_date", $rdiconsolidation_entry_date);
$statement->bindValue(":rdiconsolidation_permit_number", $rdiconsolidation_permit_number);
$statement->bindValue(":rdiconsolidation_permit_date_str", $rdiconsolidation_permit_date_str);
$statement->bindValue(":rdiconsolidation_permit_date", $rdiconsolidation_permit_date);
$statement->bindValue(":rdiconsolidation_project_addr", $rdiconsolidation_project_addr);
$statement->bindValue(":rdiconsolidation_sub_div_name", $rdiconsolidation_sub_div_name);
$statement->bindValue(":rdiconsolidation_project_type", $rdiconsolidation_project_type);
$statement->bindValue(":rdiconsolidation_company", $rdiconsolidation_company);
$statement->bindValue(":rdiconsolidation_address", $rdiconsolidation_address);
$statement->bindValue(":rdiconsolidation_city", $rdiconsolidation_city);
$statement->bindValue(":rdiconsolidation_state", $rdiconsolidation_state);
$statement->bindValue(":rdiconsolidation_zip", $rdiconsolidation_zip);
$statement->bindValue(":rdiconsolidation_phone", $rdiconsolidation_phone);
$statement->bindValue(":rdiconsolidation_project_city", $rdiconsolidation_project_city);
$statement->bindValue(":rdiconsolidation_project_state", $rdiconsolidation_project_state);
$statement->bindValue(":rdiconsolidation_size", $rdiconsolidation_size);
$statement->bindValue(":rdiconsolidation_units", $rdiconsolidation_units);
$statement->bindValue(":rdiconsolidation_building_type", $rdiconsolidation_building_type);
$statement->bindValue(":rdiconsolidation_value", $rdiconsolidation_value);
$statement->bindValue(":rdiconsolidation_work_type", $rdiconsolidation_work_type);
$statement->bindValue(":rdiconsolidation_county", $rdiconsolidation_county);
$statement->bindValue(":rdiconsolidation_lot", $rdiconsolidation_lot);
$statement->bindValue(":rdiconsolidation_dist_ll", $rdiconsolidation_dist_ll);
$statement->bindValue(":rdiconsolidation_ct1", $rdiconsolidation_ct1);
$statement->bindValue(":rdiconsolidation_owner_name", $rdiconsolidation_owner_name);
$statement->bindValue(":rdiconsolidation_owner_address", $rdiconsolidation_owner_address);
$statement->bindValue(":rdiconsolidation_owner_city", $rdiconsolidation_owner_city);
$statement->bindValue(":rdiconsolidation_owner_state", $rdiconsolidation_owner_state);
$statement->bindValue(":rdiconsolidation_owner_zip", $rdiconsolidation_owner_zip);
$statement->bindValue(":rdiconsolidation_owner_phone", $rdiconsolidation_owner_phone);
$statement->bindValue(":rdiconsolidation_ytd_permits", $rdiconsolidation_ytd_permits);
$statement->bindValue(":rdiconsolidation_ytd_value", $rdiconsolidation_ytd_value);
$statement->bindValue(":rdiconsolidation_12mth_prev_permits", $rdiconsolidation_12mth_prev_permits);
$statement->bindValue(":rdiconsolidation_date", $rdiconsolidation_date);
if (!$statement->execute()) {
$connection->rollBack();
$this->debug($connection->errorInfo()[2]);
if ($processed_record_count !== $permit_record_count) {
throw new RuntimeException("Record counts did not match. Rolling back database. Contact Support.");
}
$processed_record_count++;
}
if ($processed_record_count !== $rdi_record_count) {
$connection->rollBack();
$XSLParms["CONSOLIDATION_ERROR"] = "Record Counts Did not match. Rolling Back Database. Contact Support.";
} else {
$connection->commit();
$this->executeSQL("truncate rdis");
$connection->exec("truncate permits");
} catch (Exception $e) {
if (($connection instanceof PDO) && $connection->inTransaction()) {
$connection->rollBack();
}
$XSLParms["CONSOLIDATION-ERROR"] = $e->getMessage();
}
$XSLParms["RDI-RECORD-COUNT"] = $rdi_record_count;
$XSLParms["RDI-RECORD-COUNT"] = $permit_record_count;
$XSLParms["PROCESSED-RECORD-COUNT"] = $processed_record_count;
$content = $this->applyXSL("", $this->getXSL("consolidationResults"), $XSLParms);
$content = $this->applyXSL("", $this->getXSL("consolidationResults"), $XSLParms);
$this->displayContent($content);
break;
@@ -239,16 +167,6 @@ class Consolidation extends Base {
}
}
// ====================
// Return an RDI Object
// ====================
public function getRdiTable() {
$SQL = "SELECT * FROM permits";
return $this->getTable("rdis", $SQL);
}
}
?>
classes/Import.php
+157 -82
View File
@@ -45,7 +45,11 @@ class Import extends Base {
case "prompt":
$content = $this->applyXSL("", $this->getXSL("uploadCSVFile"));
$import_type = filter_input(INPUT_POST, "import_type") ?: "incremental";
$XSLParms['IMPORT-TYPE'] = $import_type;
$content = $this->applyXSL("", $this->getXSL("uploadCSVFile"), $XSLParms);
$this->displayContent($content);
break;
@@ -54,6 +58,7 @@ class Import extends Base {
$XSLParms['UPLOAD_STATUS'] = "UPLOAD SUCCESSFUL";
$environment = (string) $this->getSettings()->Environment ?: "Production";
$upload_type = filter_input(INPUT_POST, 'upload_type') ?: "incremental";
if (empty($environment)) {
$this->logError("Invalid Environment ({$environment}). " . __METHOD__);
@@ -63,30 +68,57 @@ class Import extends Base {
// Process the Attachment
if (!empty($_FILES)) {
if (empty($_FILES["csvfile"])) {
$this->displayNotice("No CSV file was uploaded.");
return;
}
if (!is_dir($environment_directory)) {
mkdir($environment_directory);
}
if (!is_dir($environment_directory) && !mkdir($environment_directory, 0775, true)) {
$this->displayNotice("Unable to create the upload directory. Please contact Support.");
return;
}
$tempname = $_FILES["csvfile"]["tmp_name"];
$filename = $_FILES["csvfile"]["name"];
if ($_FILES["csvfile"]["error"] !== UPLOAD_ERR_OK) {
$this->displayNotice("The CSV file upload failed. Please try again.");
return;
}
if (empty($tempname) or empty($filename)) {
return;
}
$tempname = $_FILES["csvfile"]["tmp_name"];
$filename = basename($_FILES["csvfile"]["name"]);
if (!move_uploaded_file($tempname, "{$environment_directory}/{$filename}")) {
$XSLParms['UPLOAD_STATUS'] = "UPLOAD FAILED";
}
if (empty($tempname) or empty($filename)) {
$this->displayNotice("The uploaded CSV file could not be read. Please try again.");
return;
}
$file_directory = "{$environment_directory}/{$filename}";
$import = $this->parseCSVFile($file_directory);
if (!move_uploaded_file($tempname, $file_directory)) {
$this->displayNotice("Unable to save the uploaded CSV file. Please contact Support.");
return;
}
$content = $this->applyXSL("", $this->getXSL("uploadMessage"), $XSLParms);
$this->displayContent($content);
if ($upload_type === 'incremental') {
$this->parseCSVFile($file_directory);
$content = $this->applyXSL("", $this->getXSL("uploadMessage"), $XSLParms);
$this->displayContent($content);
return;
}
if ($upload_type === 'full') {
$successful = $this->createImportRequest($file_directory);
if (!$successful) {
$this->displayNotice("An error occurred while creating the import request. Please contact Support.");
return;
}
$this->displayNotice("Your import has been queued. You will receive a notification when it is complete.");
return;
}
$this->displayNotice("Invalid upload type. Please contact Support.");
break;
}
@@ -143,73 +175,74 @@ class Import extends Base {
$County = (new Counties())->getCountiesByName($County)->record->county_serial;
$SQL = "INSERT INTO permits (
permit_permit_facts_id,
permit_entry_date,
permit_permit_number,
permit_permit_date,
permit_project_addr,
permit_sub_div_name,
permit_project_type,
permit_company,
permit_address,
permit_city,
permit_state,
permit_zip,
permit_phone,
permit_project_city,
permit_project_state,
permit_size,
permit_units,
permit_building_type,
permit_value,
permit_work_type,
permit_county,
permit_lot,
permit_dist_ll,
permit_ct1,
permit_owner_name,
permit_owner_address,
permit_owner_city,
permit_owner_state,
permit_owner_zip,
permit_owner_phone,
permit_ytd_permits,
permit_ytd_value,
permit_12mth_prev_permits )
permit_permit_facts_id,
permit_entry_date,
permit_permit_number,
permit_permit_date,
permit_project_addr,
permit_sub_div_name,
permit_project_type,
permit_company,
permit_address,
permit_city,
permit_state,
permit_zip,
permit_phone,
permit_project_city,
permit_project_state,
permit_size,
permit_units,
permit_building_type,
permit_value,
permit_work_type,
permit_county,
permit_lot,
permit_dist_ll,
permit_ct1,
permit_owner_name,
permit_owner_address,
permit_owner_city,
permit_owner_state,
permit_owner_zip,
permit_owner_phone,
permit_ytd_permits,
permit_ytd_value,
permit_12mth_prev_permits )
VALUES (
:permit_permit_facts_id,
:permit_entry_date,
:permit_permit_number,
:permit_permit_date,
:permit_project_addr,
:permit_sub_div_name,
:permit_project_type,
:permit_company,
:permit_address,
:permit_city,
:permit_state,
:permit_zip,
:permit_phone,
:permit_project_city,
:permit_project_state,
:permit_size,
:permit_units,
:permit_building_type,
:permit_value,
:permit_work_type,
:permit_county,
:permit_lot,
:permit_dist_ll,
:permit_ct1,
:permit_owner_name,
:permit_owner_address,
:permit_owner_city,
:permit_owner_state,
:permit_owner_zip,
:permit_owner_phone,
:permit_ytd_permits,
:permit_ytd_value,
:permit_12mth_prev_permits )";
:permit_permit_facts_id,
:permit_entry_date,
:permit_permit_number,
:permit_permit_date,
:permit_project_addr,
:permit_sub_div_name,
:permit_project_type,
:permit_company,
:permit_address,
:permit_city,
:permit_state,
:permit_zip,
:permit_phone,
:permit_project_city,
:permit_project_state,
:permit_size,
:permit_units,
:permit_building_type,
:permit_value,
:permit_work_type,
:permit_county,
:permit_lot,
:permit_dist_ll,
:permit_ct1,
:permit_owner_name,
:permit_owner_address,
:permit_owner_city,
:permit_owner_state,
:permit_owner_zip,
:permit_owner_phone,
:permit_ytd_permits,
:permit_ytd_value,
:permit_12mth_prev_permits )";
$statement = $connection->prepare($SQL);
@@ -255,6 +288,48 @@ class Import extends Base {
return true;
}
// =====================
// Create Import Request
// =====================
public function createImportRequest($filepath = "") {
if (!$filepath) {
return false;
}
$SQL = "insert into importjobs
( importjob_file_path,
importjob_notified,
importjob_creator,
importjob_created )
VALUES ( :importjob_file_path,
:importjob_notified,
:importjob_creator,
:importjob_created )";
$statement = $this->connect()->prepare($SQL);
$statement->bindValue(":importjob_file_path", "../{$filepath}", PDO::PARAM_STR);
$statement->bindValue(":importjob_notified", false, PDO::PARAM_BOOL);
$statement->bindValue(":importjob_creator", $this->current_user_name, PDO::PARAM_STR);
$statement->bindValue(":importjob_created", $this->getTimestamp());
if (!$statement->execute()) {
return false;
}
$to = $this->getSettings()->DeveloperEmail;
$subject = "CSV Import Jobs Queued";
$message = "Import initialized successfully. Processing will start momentarily.";
(new Email())->sendEmail($to, $subject, $message);
return true;
}
// ========================
// Return an Formatted Date
// ========================
classes/ImportLargeCSV.php
+255
View File
@@ -0,0 +1,255 @@
<?php
// ======================
// Import Large CSV Class
// ======================
class ImportLargeCSV extends Base {
// Variables
private $current_user_serial = 0;
private $current_user_id = "";
private $current_user_name = "";
private $current_user_email = "";
private $pagination_size = 20;
private $pagination_page = 1;
// =================
// Class Constructor
// =================
public function __construct() {
set_error_handler(array($this, "displayError"));
date_default_timezone_set(self::TIMEZONE);
$this->current_user_serial = $_SESSION[self::USER_SERIAL] ?? 0;
$this->current_user_id = $_SESSION[self::USER_ID] ?? "";
$this->current_user_name = $_SESSION[self::USER_NAME] ?? "";
$this->current_user_email = $_SESSION[self::USER_EMAIL] ?? "";
$this->pagination_page = $_SESSION[self::PAGINATION_PAGE] ?? 1;
}
// =====================
// Create Import Request
// =====================
public function createImportRequest() {
$subscription_serial = filter_input(INPUT_POST, "subscription_serial", FILTER_SANITIZE_NUMBER_INT) ?: 0;
$subscription = (new Subscriptions())->getSubscription($subscription_serial);
if ($subscription->count() === 0) {
$this->logError("Invalid Subscription ({$subscription_serial}): " . __METHOD__);
}
$import_type = filter_input(INPUT_POST, "import_type") ?: "";
$report_type = filter_input(INPUT_POST, "report_type") ?: "";
$importjob_import_type = str_replace(' ', '', $subscription->record->subscription_full_title) . "$import_type";
$importjob_file_name = str_replace(' ', '_', $subscription->record->subscription_full_title) . "_all_records.{$report_type}";
// $importjob_file_name = str_replace('&', 'And', $subscription->record->subscription_full_title) . "_all_records.{$report_type}";
$SQL = "insert into importjobs
( importjob_subscription,
importjob_import_type,
importjob_file_name,
importjob_file_type,
importjob_creator,
importjob_created )
VALUES ( :importjob_subscription,
:importjob_import_type,
:importjob_file_name,
:importjob_file_type,
:importjob_creator,
:importjob_created )";
$statement = $this->connect()->prepare($SQL);
$statement->bindValue(":importjob_subscription", $subscription_serial);
$statement->bindValue(":importjob_import_type", $importjob_import_type);
$statement->bindValue(":importjob_file_name", $importjob_file_name);
$statement->bindValue(":importjob_file_type", strtoupper($report_type));
$statement->bindValue(":importjob_creator", $this->current_user_serial);
$statement->bindValue(":importjob_created", $this->getTimestamp());
$statement->execute();
$this->displayNotice("Your import has been queued. You will receive an notification when it is ready.");
}
// ============
// User Imports
// ============
public function usersImports() {
$imports = $this->getUsersImports();
$popovers = (new Popovers())->getPopovers();
$this->markImportsAsSeen();
$XML = $this->mergeXML($imports, "<XML/>");
$XML = $this->mergeXML($popovers, $XML);
$html = $this->applyXSL($XML, $this->getXSL("viewImports"));
$this->displayContent($html);
}
// =========================
// Mark User Imports as Seen
// =========================
public function markImportsAsSeen() {
$SQL = "update importjobs
set importjob_notified = true
where importjobs.importjob_creator = {$this->current_user_serial}";
$statement = $this->connect()->prepare($SQL);
$statement->execute();
}
// =======================
// Return a Imports Object
// =======================
public function getUsersImports() {
// Find Criteria
$find_importjob = filter_input(INPUT_POST, "find_importjob") ?: "";
$find_words = $this->sanitizeString($find_importjob);
// Show Per Page
$this->pagination_size = filter_input(INPUT_POST, "importjobs_per_page", FILTER_SANITIZE_NUMBER_INT) ?: $this->pagination_size;
// Record Count
$connection = $this->connect();
$SQL = "select count(*) as count
from view_importjobs
where view_importjobs.importjob_creator = {$this->current_user_serial}
and view_importjobs.subscription_full_title regexp :subscription_full_title";
$statement = $connection->prepare($SQL);
$statement->bindValue(":subscription_full_title", $find_words);
$statement->execute();
$recordset = $statement->fetch(PDO::FETCH_ASSOC);
$count = (count($recordset) > 0) ? (integer) $recordset["count"] : (integer) 0;
// Pagination
$pagination = filter_input(INPUT_POST, "pagination") ?: "";
switch ($pagination) {
case "first" : $this->pagination_page = 1;
break;
case "previous" : $this->pagination_page = max(($this->pagination_page - 1), 1);
break;
case "next" : $this->pagination_page = ($this->pagination_page + 1);
break;
case "last" : $this->pagination_page = ($count / $this->pagination_size);
break;
}
$offset = (($this->pagination_page * $this->pagination_size) - $this->pagination_size);
$start = ($offset + 1);
$end = min($count, (($start + $this->pagination_size) - 1));
// Data
$SQL = "select view_importjobs.*
from view_importjobs
where view_importjobs.importjob_creator = {$this->current_user_serial}
and view_importjobs.subscription_full_title regexp :subscription_full_title
limit {$this->pagination_size}
offset {$offset}";
$statement = $connection->prepare($SQL);
$statement->bindValue(":subscription_full_title", $find_words);
$statement->execute();
$importjobs = $this->getTableXML("importjobs", $statement);
// Insert Pagination Attributes
$importjobs->addAttribute("count", $count);
$importjobs->addAttribute("start", $start);
$importjobs->addAttribute("end", $end);
$importjobs->addAttribute("page", $this->pagination_page);
$_SESSION[self::PAGINATION_PAGE] = $this->pagination_page;
return $importjobs;
}
// =====================================
// Return if users imports are completed
// =====================================
public function checkUsersCompletedImports_AJAX() {
$SQL = "select count(*) as notify
from view_importjobs
where view_importjobs.importjob_creator = :current_user_serial
and view_importjobs.importjob_status = 'complete'
and view_importjobs.importjob_notified = 0";
$importjobs = $this->getTable("importjobs", $SQL, array('current_user_serial' => $this->current_user_serial));
// Create the User Object.
$users_imports = array();
$users_imports["notify"] = ($importjobs->record->notify == 0) ? false : true;
if ($_SESSION[self::AJAX]) {
echo json_encode($users_imports);
} else {
return $users_imports;
}
}
// ===============
// Download Import
// ===============
public function downloadImport() {
$import_code = filter_input(INPUT_POST, "import_code") ?: "";
$SQL = "select view_importjobs.*
from view_importjobs
where view_importjobs.importjob_download_token = :import_code
and view_importjobs.importjob_status = 'complete' limit 1";
$importjob = $this->getTable("importjob", $SQL, array("import_code" => $import_code));
$file_path = str_replace('../imports/', './imports/', $importjob->record->importjob_file_path);
$file_name = $importjob->record->importjob_file_name;
header('Content-Type: application/octet-stream');
header("Content-Disposition: attachment; filename=\"$file_name\"");
readfile($file_path);
}
}
?>
classes/Process_ImportRequest.php
+584
View File
@@ -0,0 +1,584 @@
<?php
declare(strict_types=1);
$basePath = realpath(__DIR__ . '/..');
require_once $basePath . '/vendor/autoload.php';
use PHPMailer\PHPMailer\PHPMailer;
use PHPMailer\PHPMailer\Exception;
// =====================
// Process_ImportRequest
// =====================
class Process_ImportRequest {
// Variables
private $log = "";
private $settings = "";
private $connection = "";
// ==================
// Class Constructor.
// ==================
public function __construct() {
$basePath = realpath(__DIR__ . '/..');
$logfile = $basePath . "/scripts/importrequests.log";
// ==========================
// Log Rotation (50 MB limit)
// ==========================
$maxLogSize = 50 * 1024 * 1024; // 50 MB
if (file_exists($logfile) && filesize($logfile) >= $maxLogSize) {
$rotated = $logfile . '-';
rename($logfile, $rotated);
}
$this->log = fopen($logfile, "a");
if ($this->log === false) {
error_log("Process_ImportRequest : Unable to open log file");
exit();
}
$this->log(str_repeat("*", 50));
// Get Settings
$this->settings = new SimpleXMLElement((file_exists($basePath . "/xml/settings.xml") ? file_get_contents($basePath . "/xml/settings.xml") : "<settings/>"));
if ($this->settings->count() === 0) {
error_log("Process_ImportRequest : Unable to initialize settings");
exit();
}
// Open Database Connections
$this->connection = $this->connect();
}
// ================================================
// Return a New DEC-DR Database Database Connection
// ================================================
public function connect() {
if ($this->settings->Environment == 'Production') {
$host = 'localhost';
} else {
$host = '192.168.1.190';
}
try {
$database = (string) $this->settings->DatabaseName;
$user = (string) $this->settings->DatabaseUser;
$password = (string) $this->settings->DatabasePassword;
$connection = new PDO("mysql:host={$host};dbname={$database};charset=utf8", $user, $password);
$connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false);
$connection->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
} catch (PDOException | Exception $e) {
trigger_error($e->getMessage());
}
return $connection;
}
// =============================
// Log a message to the Log File
// =============================
private function log($message = "") {
fwrite($this->log, date("Y-m-d H:i A") . " : " . "{$message} \n");
}
// ===================
// Process Parcel Data
// ===================
public function process() {
$this->log("Starting All Record Import Job");
$this->runImportJob();
$this->purgeImportJobs();
$this->log("Completed All Record Import Job");
}
// ==============
// Run Import Job
// ==============
private function runImportJob() {
$this->log("Processing Import Job");
$nextjob = $this->claimNextJob();
if (!$nextjob) {
$this->log("Nothing to do.");
return;
}
$importjob_serial = (int) $nextjob['importjob_serial'];
$importjob_file_path = (string) $nextjob['importjob_file_path'];
try {
$success = $this->parseCSVFile($importjob_file_path);
$SQL = "UPDATE importjobs
SET importjob_status = 'complete',
importjob_completed = NOW()
WHERE importjob_serial = :importjob_serial ";
$update = $this->connect()->prepare($SQL);
$update->execute([':importjob_serial' => $importjob_serial]);
} catch (Throwable $e) {
$SQL = "UPDATE importjobs
SET importjob_status = 'failed',
importjob_completed = NOW(),
importjob_error_message = :importjob_error_message
WHERE importjob_serial = :importjob_serial";
$fail = $this->connect()->prepare($SQL);
$fail->execute([':importjob_serial' => $importjob_serial, ':importjob_error_message' => $e->getMessage()]);
// Notify Admins of error
$this->sendWebHookAlert("Import Job {$importjob_serial} has failed. Check Error Logs for details.");
$this->email_admins("Import Job Failure.", "Import Job {$importjob_serial} has failed. Check Error Logs for details.");
}
}
// =================================
// Purge Imports Older then 24 Hours
// =================================
private function purgeImportJobs() {
$this->log("Purging Import Jobs Database Files");
$this->purgeFiles();
}
// =======================
// Claim Next Job in Queue
// =======================
private function claimNextJob() {
$this->connection->beginTransaction();
$SQL = "SELECT view_importjobs.*
FROM view_importjobs
WHERE view_importjobs.importjob_status = 'queued'
ORDER BY view_importjobs.importjob_created ASC
LIMIT 1 FOR UPDATE";
$statement = $this->connection->prepare($SQL);
$statement->execute();
$job = $statement->fetch(PDO::FETCH_ASSOC);
if (!$job) {
$this->connection->commit();
return null;
}
$SQL = "UPDATE importjobs
SET importjob_status = 'running'
WHERE importjob_serial = :importjob_serial";
$statement = $this->connection->prepare($SQL);
$statement->execute([':importjob_serial' => (int) $job['importjob_serial']]);
$this->connection->commit();
return $job;
}
// ====================
// Purges Files on Disk
// ====================
private function purgeFiles() {
$SQL = "select *
from importjobs
where importjobs.importjob_created < (NOW() - INTERVAL 24 HOUR)";
$importfiles = $this->connect()->query($SQL)->fetchAll(PDO::FETCH_ASSOC);
if (count($importfiles) == 0) {
$this->log("Nothing To Purge");
return;
}
foreach ($importfiles as $files) {
$filename = $files['importjob_file_path'];
unlink($filename);
}
$baseDir = './imports';
foreach (new DirectoryIterator($baseDir) as $item) {
if ($item->isDot() || !$item->isDir()) {
continue;
}
$folderPath = $item->getPathname();
// Check if directory is empty
$files = scandir($folderPath);
if ($files !== false && count($files) === 2) {
rmdir($folderPath);
}
}
$this->log("Purging Import Jobs Database Records");
$SQL = "delete from importjobs where importjobs.importjob_created < (NOW() - INTERVAL 24 HOUR)";
$this->connection->query($SQL);
}
// =======================
// Format Raw Phone Number
// =======================
public function formatPhoneNumber(string $phonenumber): string {
// Remove everything except digits
$digits = preg_replace('/\D+/', '', $phonenumber);
// Handle leading country code (US)
if (strlen($digits) === 11 && str_starts_with($digits, '1')) {
$digits = substr($digits, 1);
}
return substr($digits, 0, 3) . '-' . substr($digits, 3, 3) . '-' . substr($digits, 6, 4);
}
// ==============
// Parse CSV File
// ==============
public function parseCSVFile($file) {
$csvfile = fopen($file, 'r');
$RecordCount = 0;
while (($data = fgetcsv($csvfile, 0, ",", '"', '\\')) !== FALSE) {
$PermitFactsId = $data[1];
$EntryDate = $this->fixDate($data[2]);
$PermitNumber = $data[4];
$PermitDate = $this->fixDate($data[5]);
$ProjectAddress = $data[7];
$SubdivisionName = $data[8];
$ProjectType = $data[9];
$Company = substr($data[10], 0, 30);
$Address = substr($data[11], 0, 30);
$City = $data[12];
$State = $data[13];
$Zip = $data[14];
$Phone = $data[15];
$ProjectCity = $data[16];
$ProjectState = $data[17];
$Size = $data[18] ?: 0;
$Units = $data[19];
$BuildingType = $data[20];
$Value = $data[21] ?: 0;
$WorkType = $data[22];
$County = $data[23] ?: 0;
$Lot = $data[24];
$DistrictLL = $data[25];
$Ctl = $data[26];
$OwnerName = substr($data[27], 0, 30);
$OwnerAddress = substr($data[28], 0, 30);
$OwnerCity = $data[29];
$OwnerState = substr($data[30], 0, 2);
$OwnerZip = $data[31];
$OwnerPhone = $data[32];
$YTDPermits = $data[33] ?: 0;
$YTDValue = $data[34] ?: 0;
$TwelveMonthPreviousPermits = $data[35] ?: 0;
$County = $this->getCountiesByName($County)['county_serial'] ?? 0;
$SQL = "INSERT INTO permits (
permit_permit_facts_id,
permit_entry_date,
permit_permit_number,
permit_permit_date,
permit_project_addr,
permit_sub_div_name,
permit_project_type,
permit_company,
permit_address,
permit_city,
permit_state,
permit_zip,
permit_phone,
permit_project_city,
permit_project_state,
permit_size,
permit_units,
permit_building_type,
permit_value,
permit_work_type,
permit_county,
permit_lot,
permit_dist_ll,
permit_ct1,
permit_owner_name,
permit_owner_address,
permit_owner_city,
permit_owner_state,
permit_owner_zip,
permit_owner_phone,
permit_ytd_permits,
permit_ytd_value,
permit_12mth_prev_permits )
VALUES (
:permit_permit_facts_id,
:permit_entry_date,
:permit_permit_number,
:permit_permit_date,
:permit_project_addr,
:permit_sub_div_name,
:permit_project_type,
:permit_company,
:permit_address,
:permit_city,
:permit_state,
:permit_zip,
:permit_phone,
:permit_project_city,
:permit_project_state,
:permit_size,
:permit_units,
:permit_building_type,
:permit_value,
:permit_work_type,
:permit_county,
:permit_lot,
:permit_dist_ll,
:permit_ct1,
:permit_owner_name,
:permit_owner_address,
:permit_owner_city,
:permit_owner_state,
:permit_owner_zip,
:permit_owner_phone,
:permit_ytd_permits,
:permit_ytd_value,
:permit_12mth_prev_permits )";
$statement = $this->connection->prepare($SQL);
$statement->bindValue(":permit_permit_facts_id", $PermitFactsId);
$statement->bindValue(":permit_permit_number", $PermitNumber);
$statement->bindValue(":permit_entry_date", $EntryDate);
$statement->bindValue(":permit_permit_date", $PermitDate);
$statement->bindValue(":permit_project_addr", $ProjectAddress);
$statement->bindValue(":permit_sub_div_name", $SubdivisionName);
$statement->bindValue(":permit_project_type", $ProjectType);
$statement->bindValue(":permit_company", $Company);
$statement->bindValue(":permit_address", $Address);
$statement->bindValue(":permit_city", $City);
$statement->bindValue(":permit_state", $State);
$statement->bindValue(":permit_zip", $Zip);
$statement->bindValue(":permit_phone", $Phone);
$statement->bindValue(":permit_project_city", $ProjectCity);
$statement->bindValue(":permit_project_state", $ProjectState);
$statement->bindValue(":permit_size", $Size);
$statement->bindValue(":permit_units", $Units);
$statement->bindValue(":permit_building_type", $BuildingType);
$statement->bindValue(":permit_value", $Value);
$statement->bindValue(":permit_work_type", $WorkType);
$statement->bindValue(":permit_county", $County);
$statement->bindValue(":permit_lot", $Lot);
$statement->bindValue(":permit_dist_ll", $DistrictLL);
$statement->bindValue(":permit_ct1", $Ctl);
$statement->bindValue(":permit_owner_name", $OwnerName);
$statement->bindValue(":permit_owner_address", $OwnerAddress);
$statement->bindValue(":permit_owner_city", $OwnerCity);
$statement->bindValue(":permit_owner_state", $OwnerState);
$statement->bindValue(":permit_owner_zip", $OwnerZip);
$statement->bindValue(":permit_owner_phone", $OwnerPhone);
$statement->bindValue(":permit_ytd_permits", $YTDPermits);
$statement->bindValue(":permit_ytd_value", $YTDValue);
$statement->bindValue(":permit_12mth_prev_permits", $TwelveMonthPreviousPermits);
$statement->execute();
$RecordCount++;
}
return true;
}
// ========================
// Return a Counties Object
// ========================
public function getCountiesByName($county_name) {
$SQL = "select view_counties.*
from view_counties
where view_counties.county_name = '{$county_name}'";
$statement = $this->connection->prepare($SQL);
$statement->execute();
$county_name = $statement->fetch(PDO::FETCH_ASSOC);
return $county_name;
}
// ==========================
// Send an Alert/Notification
// ==========================
public function sendWebHookAlert($message, $title = "🚨 DEC Application Alert", $alerttype = "1fff00") {
$webhookUrl = (string) $this->settings->DiscordWebHookURL;
$data = [
'embeds' => [[
'title' => $title,
'description' => $message,
'color' => hexdec($alerttype),
'footer' => [
'text' => 'Real Estate Data Inc. System Notificiation',
],
'timestamp' => date('c'), // ISO 8601 format
]]
];
$jsonData = json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
$ch = curl_init($webhookUrl);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $jsonData);
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
$response = curl_exec($ch);
if ($response === false) {
$this->log('Curl error: ' . curl_error($ch));
}
curl_close($ch);
}
// ============
// Email Admins
// ============
public function email_admins($subject = "", $message = "") {
$this->log(date("Y-m-d H:i:s") . " - Emailing Admins...");
$settings = new SimpleXMLElement((file_exists("./xml/settings.xml") ? file_get_contents("./xml/settings.xml") : "<settings/>"));
// Server Variables
$host = (string) $settings->SmtpServer;
$username = (string) $settings->SmtpUserName;
$password = (string) $settings->SmtpPassword;
// Recipient Variables
$from_email = (string) $settings->FromEmail;
$from_name = (string) $settings->FromName;
$support_email = (string) $settings->SupportEmail;
$support_name = (string) $settings->SupportName;
if ((string) $settings->Environment == 'Production') {
$admin_email = (string) $settings->AdminEmail;
$admin_name = (string) $settings->AdminName;
}
// Content Variables
$attachment_name = "";
$mail = new PHPMailer(true);
try {
//Server settings
$mail->isSMTP();
$mail->Host = $host;
$mail->SMTPAuth = true;
$mail->Username = $username;
$mail->Password = $password;
$mail->SMTPSecure = PHPMailer::ENCRYPTION_SMTPS;
$mail->Port = 465;
//Recipients
$mail->setFrom($from_email, $from_name);
$mail->addAddress($support_email, $support_name);
if ($settings->Environment == 'Production') {
$mail->addAddress($admin_email, $admin_name);
}
//Attachments
$mail->addAttachment($attachment_name);
//Content
$mail->isHTML(true);
$mail->Subject = $subject;
$mail->Body = $message;
$mail->send();
} catch (Exception $e) {
$this->log("Message could not be sent. Mailer Error: {$e->ErrorInfo}");
}
}
// ========================
// Return an Formatted Date
// ========================
public function fixDate($givenDate) {
$formattedDate = date('Y-m-d', strtotime($givenDate));
return $formattedDate;
}
}
?>
classes/Subscriptions.php
+23 -7
View File
@@ -321,10 +321,12 @@ class Subscriptions extends Base {
$user_serial = filter_input(INPUT_POST, "user_reference", FILTER_SANITIZE_NUMBER_INT) ?: 0;
$user = (new Users())->getUser($user_serial);
$usersubscriptions = $this->getUsersActiveSubscriptions($user_serial);
$subscriptions = $this->getSubscriptions();
$XML = $this->mergeXML($user, "<XML/>");
$XML = $this->mergeXML($subscriptions, $XML);
$XML = $this->mergeXML($usersubscriptions, $XML);
$content = $this->applyXSL($XML, $this->getXSL("addSubscriptionToUser"));
@@ -334,11 +336,18 @@ class Subscriptions extends Base {
case "add":
$subscription_serial = filter_input(INPUT_POST, "subscription_serial", FILTER_SANITIZE_NUMBER_INT) ?: 0;
$user_serial = filter_input(INPUT_POST, "user_serial", FILTER_SANITIZE_NUMBER_INT) ?: 0;
$subscription_stop_date = filter_input(INPUT_POST, "subscription_stop_date") ?: "";
$subscription_serials = filter_input(INPUT_POST, "subscriptions", FILTER_SANITIZE_NUMBER_INT, FILTER_REQUIRE_ARRAY) ?: array();
$subscription_end_dates = filter_input(INPUT_POST, "subscription_end_date", FILTER_DEFAULT, FILTER_REQUIRE_ARRAY) ?: array();
$subscriptions = array();
$subscription_stop_date = date("Y-m-d", strtotime($subscription_stop_date));
foreach ($subscription_serials as $index => $subscription_serial) {
$subscriptions[] = array(
"usersubscription_subscription" => $subscription_serial,
"usersubscription_end_date" => date("Y-m-d", strtotime($subscription_end_dates[$index] ?? ""))
);
}
$SQL = "insert into usersubscriptions
@@ -357,12 +366,19 @@ class Subscriptions extends Base {
$statement = $this->connect()->prepare($SQL);
$statement->bindValue(":usersubscription_user_serial", $user_serial);
$statement->bindValue(":usersubscription_subscription_serial", $subscription_serial);
$statement->bindValue(":usersubscription_stop_date", $subscription_stop_date);
$statement->bindValue(":usersubscription_timestamp", $this->getTimestamp());
$statement->bindValue(":usersubscription_origin", $this->current_user_name);
$statement->execute();
foreach ($subscriptions as $subscription) {
$subscription_serial = $subscription['usersubscription_subscription'];
$subscription_stop_date = $subscription['usersubscription_end_date'];
$statement->bindValue(":usersubscription_subscription_serial", $subscription_serial);
$statement->bindValue(":usersubscription_stop_date", $subscription_stop_date);
$statement->execute();
}
break;
@@ -583,7 +599,7 @@ class Subscriptions extends Base {
$XML = $this->mergeXML($subscriptions, "<XML/>");
$XML = $this->mergeXML($alerts, $XML);
$XML = $this->mergeXML($popovers, $XML);
$content = $this->applyXSL($XML, $this->getXSL("userSubscriptions"));
$this->displayContent($content);
classes/Users.php
+14
View File
@@ -743,6 +743,20 @@ class Users extends Base {
error_log($statement->errorInfo()[2]);
}
}
// ====================================
// Return a Users Login Activity Object
// ====================================
public function getLoginActivity($userserial) {
$SQL = "select view_journal.journal_timestamp_verbose, view_journal.journal_ip
from view_journal
where view_journal.journal_origin = '{$userserial}'
and view_journal.journal_entry like '{$userserial} %logged in%'";
return $this->getTable("loginactivity", $SQL);
}
}
?>