Funeral Notices

2 Part import one for the listings and one for the details. 

Functions extracted from imports.extend

Crons

#notices list import below
*/28 * * * * wget --spider https://api.kruxor.com/imports/3zJFx/ > /dev/null 2>&1
#notices details import below
*/14 * * * * wget --spider https://api.kruxor.com/imports/4aZls/ > /dev/null 2>&1

Notice Listing

https://api.kruxor.com/list/notices/ 

Part 1 List Import

Grabs the details from the listing page. 

  /*

  Import list of items from : https://www.mytributes.com.au/all-locations/?sort=date
  Running import type: import_notices_list into class extend notices

  */
  public function import_notices_list() {
      /* This will be run on a loaded import item, so dont need to pass variabled to the function */
      global $db;
      global $functions;
      $out = "";
      $for_counter = 0;
      $main_loop_tag = ".notice-board-list .notice-message";
      $db_table_name = $this->db->escapeString($this->db_table_name);
      $loop_max = 100;

      // import url is a bit different, need to replace the search part of the string with search_term
      // https://api.kruxor.com/imports/GMJQx/?search_term=new+world
      // test html -- https://api.kruxor.com/steam_test.html

      // $search_term = $_GET['search_term'];
      // $search_term_enc = urlencode($search_term);

      $import_url = $this->import_url;
      // $import_url_search = str_replace ( "[@search_term]" , $search_term_enc , $import_url_search); // replace... [@search_term]

      //$bugs .= $import_url_search;
      //$import_url_search;

      // $functions->cache_results($id = md5($game_url), $url = $game_url, $cache_length_hours = 24, $use_simple_html = false);
      // $html_cache = "cache/". md5($game_url) . "_cache.html"; // rather than loading the url over and over from the server.

      // save the html file for testing to see if the grabbed file is the same as the one in browser.
      // file_put_contents ( "./steam_test.html" , file_get_contents($import_url_search) );

      require_once("lib/simple_html_dom.php");
      $html = file_get_html($import_url);

        foreach($html->find($main_loop_tag) as $item) {

            $for_counter++;
            if($for_counter == $loop_max) {
              continue;
            }

            $target_thread_class = new $this->target_thread_class;
            $target_thread_class->add_to_menu = false;
            $target_thread_class->start();

            /* title */
            $imp_title = $item->find('.listing-header',0)->plaintext;
            $imp_title = trim($imp_title);
            // $target_thread_class->title = $imp_title;

            /* md5 */
            $imp_title_md5 = md5($imp_title);

            /* publication */
            $imp_pub = $item->find('.metadata span',0)->plaintext; // .metadata span
            $imp_pub = trim($imp_pub);

            /* date_listed */
            $imp_date_listed = $item->find('.metadata span',1)->plaintext; // .metadata span
            $imp_date_listed = trim($imp_date_listed);

            /* link_url */
            $imp_link_url = $item->find('a',0)->href;
            $imp_link_url = "https://www.mytributes.com.au" . $imp_link_url;

            /* Names Split */
            $imp_names = explode(",",$imp_title);
            $imp_surname = $imp_names[0];
            $imp_surname = trim($imp_surname);
            $imp_other_names = $imp_names[1];
            $imp_other_names = trim($imp_other_names);

            /* attr example */
            // $imp_search_review_summary = $item->find('.search_review_summary',0)->attr['data-tooltip-html']; // attr data-tooltip-html

            /* check isset attr */
            /*
            if(isset($item->attr['data-ds-appid'])) {
              $imp_game_id = $item->attr['data-ds-appid'];
              if($imp_game_id > "") {
                $imp_game_discussions = "https://steamcommunity.com/app/".$imp_game_id."/discussions/0/";
              }
            }
            */

            // use this md5_exists
            if($target_thread_class->md5_exists($imp_title_md5)) {
              // continue on to processing replies for this thread.
            } else {
              // add as a new main thread.
              $target_thread_class->title = $imp_title;
              $target_thread_class->md5 = $imp_title_md5;
              $target_thread_class->date_listed = $imp_date_listed;
              $target_thread_class->processed = "false";
              $target_thread_class->surname = $imp_surname;
              $target_thread_class->other_name = $imp_other_names;
              $target_thread_class->publication = $imp_pub;
              $target_thread_class->link_url = $imp_link_url;
              // $target_thread_class->location = $imp_location;
              /* working here... \*/

              $target_thread_class->add();
              $target_thread_class->load_from_id($target_thread_class->id);
            }

            // just process one to test...
            //return;

        }

        return $out;

  }

 

Part 2 Details Import

This loads and imports the latest unprocessed item, if it does not find one it just returns. 

  public function import_notices_details() {
      global $db;
      global $functions;
      $out = "";
      $for_counter = 0;
      $main_loop_tag = ".listing";
      $db_table_name = $this->db->escapeString($this->db_table_name);
      $loop_max = 100;

      $notices = new notices;
      $notices->add_to_menu = false;
      $notices->start();

      $fields_array = [
        "processed" => "false",
      ];
      if(!$notices->load_from_fields_array($fields_array, $max = 1)) {
        return "nothing to load";
      }

      // we should have a loded unprocessed item now.
      $out .= $notices->title . "<br />";

      $import_url = $notices->link_url;

      require_once("lib/simple_html_dom.php");
      $html = file_get_html($import_url);

      //$item = $html->find($main_loop_tag);
        //foreach($html->find($main_loop_tag) as $item) {

            /*
            $for_counter++;
            if($for_counter == $loop_max) {
              continue;
            }
            */

            // echo "in item loop";

            //$target_thread_class = new $this->target_thread_class;
            //$target_thread_class->add_to_menu = false;
            //$target_thread_class->start();

            /* description full html */
            $imp_additional = "";
            $imp_additional = $html->find('.listing .listing-description',0)->innertext;
            $imp_additional = trim($imp_additional);
            // $target_thread_class->title = $imp_title;

            $imp_text_details = strip_tags($imp_additional);

            /* Date Range */
            $img_date_range = "";
            $imp_date_range = $html->find('.listing .listing-description p span',0)->innertext;
            $imp_date_range = trim($imp_date_range);

            /* Location */
            // short-address
            $imp_location = "";
            $imp_location = $html->find('.listing .short-address',0)->plaintext;
            $imp_location = trim($imp_location);

            // image
            $imp_img = "";
            //if(isset($item->find('img',0))) {
            if(null !== $html->find('.galleria-stage img',0)) {
              $imp_img = $html->find('.galleria-stage img',0)->src;
            }

            /* attr example */
            // $imp_search_review_summary = $item->find('.search_review_summary',0)->attr['data-tooltip-html']; // attr data-tooltip-html

            /* check isset attr */
            /*
            if(isset($item->attr['data-ds-appid'])) {
              $imp_game_id = $item->attr['data-ds-appid'];
              if($imp_game_id > "") {
                $imp_game_discussions = "https://steamcommunity.com/app/".$imp_game_id."/discussions/0/";
              }
            }
            */

            $notices->processed = "true";
            $notices->text_details = $imp_text_details;
            $notices->date_range = $imp_date_range;
            $notices->image_source = $imp_img;
            $notices->location = $imp_location;
            $notices->additional = $imp_additional;
            $notices->update();

        //}

        return $out;

  }