summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoremkael <emkael@tlen.pl>2019-01-31 17:43:10 +0100
committeremkael <emkael@tlen.pl>2019-01-31 19:41:51 +0100
commit5df7a81d952c9aed7c9adadadecdeb84f1e5c264 (patch)
treeb357b463e2d29628cd1739cd442c956b0be79d15
parent4ec34ae1ea35a21d569115cd0e98450d084e5ac4 (diff)
Universal classes for XML/HTML scraping
-rw-r--r--providers/HtmlFeed.php44
-rw-r--r--providers/XmlFeed.php39
2 files changed, 83 insertions, 0 deletions
diff --git a/providers/HtmlFeed.php b/providers/HtmlFeed.php
new file mode 100644
index 0000000..411cddf
--- /dev/null
+++ b/providers/HtmlFeed.php
@@ -0,0 +1,44 @@
+<?php
+
+namespace Providers;
+
+require_once('XmlFeed.php');
+require_once('../lib/querypath/src/qp.php');
+
+abstract class HtmlFeed extends \Providers\XmlFeed {
+
+ const HTML_FEED_USER_AGENT = 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)';
+
+ private $_encoding;
+
+ private function __getHttpContent($feedUrl) {
+ $header = array();
+ $header[] = 'Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5';
+ $header[] = 'Cache-Control: max-age=0';
+ $header[] = 'Connection: keep-alive';
+ $header[] = 'Keep-Alive: 300';
+ $header[] = 'Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7';
+ $header[] = 'Pragma: ';
+ $ch = curl_init($feedUrl);
+ curl_setopt($ch, CURLOPT_USERAGENT, self::HTML_FEED_USER_AGENT);
+ curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
+ curl_setopt($ch, CURLOPT_AUTOREFERER, TRUE);
+ curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
+ curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
+ curl_setopt($ch, CURLOPT_ENCODING, '');
+ curl_setopt($ch, CURLOPT_TIMEOUT, 20);
+ $page = curl_exec($ch);
+ curl_close ($ch);
+ $this->_encoding = mb_detect_encoding($page);
+ return $page;
+ }
+
+ protected function _fetchItems() {
+ $page = $this->__getHttpContent($this->_feedUrl);
+ $this->_feedXml = htmlqp($page, NULL, ['convert_from_encoding' => $this->_encoding, 'convert_to_encoding' => $this->_encoding]);
+ return $this->_parseFeedContent($this->_feedXml);
+ }
+
+}
+
+?>
diff --git a/providers/XmlFeed.php b/providers/XmlFeed.php
new file mode 100644
index 0000000..d5ecd1c
--- /dev/null
+++ b/providers/XmlFeed.php
@@ -0,0 +1,39 @@
+<?php
+
+namespace Providers;
+
+require_once('Provider.php');
+
+abstract class XmlFeed extends \Providers\Provider {
+
+ protected $_feedUrl;
+ protected $_feedXml;
+
+ abstract protected function _getFeedUrl($feed);
+
+ public function __construct($feed, $options=[]) {
+ parent::__construct($feed, $options);
+ $this->_feedUrl = $this->_getFeedUrl($feed);
+ if (!$this->_feedUrl) {
+ throw new Exception('XML feed "' . $feed . '" undefined');
+ }
+ }
+
+ abstract protected function _parseFeedContent($feed);
+
+ protected function _fetchItems() {
+ $this->_feedXml = new \SimpleXMLElement($this->_feedUrl, 0, TRUE);
+ return $this->_parseFeedContent($this->_feedXml);
+ }
+
+ protected function _spamFilter($content) {
+ return $content;
+ }
+
+ protected function _sortContent($content) {
+ return $content;
+ }
+
+}
+
+?>