Merge branch 'sql-optimisation' into dev

app/controllers/feedController.php

@@ -21,6 +21,7 @@ class feedController extends ActionController {
 				$def_cat = $this->catDAO->getDefault ();
 				$cat = $def_cat->id ();
 			}
+
 			$user = Request::param ('username');
 			$pass = Request::param ('password');
 			$params = array ();

app/models/Category.php

@@ -4,11 +4,22 @@ class Category extends Model {
 	private $id = false;
 	private $name;
 	private $color;
+	private $nbFeed = -1;
+	private $nbNotRead = -1;
 	private $feeds = null;
 
-	public function __construct ($name = '', $color = '#0062BE') {
+	public function __construct ($name = '', $color = '#0062BE', $feeds = null) {
 		$this->_name ($name);
 		$this->_color ($color);
+		if (!empty($feeds)) {
+			$this->_feeds ($feeds);
+			$this->nbFeed = 0;
+			$this->nbNotRead = 0;
+			foreach ($feeds as $feed) {
+				$this->nbFeed++;
+				$this->nbNotRead += $feed->nbNotRead ();
+			}
+		}
 	}
 
 	public function id () {
@@ -25,17 +36,31 @@ class Category extends Model {
 		return $this->color;
 	}
 	public function nbFeed () {
+		if ($this->nbFeed < 0) {
 		$catDAO = new CategoryDAO ();
-		return $catDAO->countFeed ($this->id ());
+			$this->nbFeed = $catDAO->countFeed ($this->id ());
+		}
+
+		return $this->nbFeed;
 	}
 	public function nbNotRead () {
+		if ($this->nbNotRead < 0) {
 		$catDAO = new CategoryDAO ();
-		return $catDAO->countNotRead ($this->id ());
+			$this->nbNotRead = $catDAO->countNotRead ($this->id ());
+		}
+
+		return $this->nbNotRead;
 	}
 	public function feeds () {
 		if (is_null ($this->feeds)) {
 			$feedDAO = new FeedDAO ();
 			$this->feeds = $feedDAO->listByCategory ($this->id ());
+			$this->nbFeed = 0;
+			$this->nbNotRead = 0;
+			foreach ($this->feeds as $feed) {
+				$this->nbFeed++;
+				$this->nbNotRead += $feed->nbNotRead ();
+			}
 		}
 
 		return $this->feeds;
@@ -150,13 +175,24 @@ class CategoryDAO extends Model_pdo {
 		}
 	}
 
-	public function listCategories () {
+	public function listCategories ($prePopulateFeeds = true) {	//TODO: Search code-base for places where $prePopulateFeeds should be false
+		if ($prePopulateFeeds) {
+			$sql = 'SELECT c.id as c_id, c.name as c_name, c.color as c_color, count(e.id) as nbNotRead, f.* '
+			     . 'FROM  ' . $this->prefix . 'category c '
+			     . 'LEFT OUTER JOIN ' . $this->prefix . 'feed f ON f.category = c.id '
+			     . 'LEFT OUTER JOIN  ' . $this->prefix . 'entry e ON e.id_feed = f.id AND e.is_read = 0 '
+			     . 'GROUP BY f.id '
+			     . 'ORDER BY c.name, f.name';
+			$stm = $this->bd->prepare ($sql);
+			$stm->execute ();
+			return HelperCategory::daoToCategoryPrepopulated ($stm->fetchAll (PDO::FETCH_ASSOC));
+		} else {
 			$sql = 'SELECT * FROM ' . $this->prefix . 'category ORDER BY name';
 			$stm = $this->bd->prepare ($sql);
 			$stm->execute ();
-
 			return HelperCategory::daoToCategory ($stm->fetchAll (PDO::FETCH_ASSOC));
 		}
+	}
 
 	public function getDefault () {
 		$sql = 'SELECT * FROM ' . $this->prefix . 'category WHERE id="000000"';
@@ -220,6 +256,54 @@ class CategoryDAO extends Model_pdo {
 }
 
 class HelperCategory {
+	public static function findFeed($categories, $feed_id) {
+		foreach ($categories as $category) {
+			foreach ($category->feeds () as $feed) {
+				if ($feed->id () === $feed_id) {
+					return $feed;
+				}
+			}
+		}
+		return null;
+	}
+
+	public static function daoToCategoryPrepopulated ($listDAO) {
+		$list = array ();
+
+		if (!is_array ($listDAO)) {
+			$listDAO = array ($listDAO);
+		}
+
+		$previousLine = null;
+		$feedsDao = array();
+		$nbLinesMinus1 = count($listDAO) - 1;
+		for ($i = 0; $i <= $nbLinesMinus1; $i++) {
+			$line = $listDAO[$i];
+			$cat_id = $line['c_id'];
+			if (($i > 0) && (($cat_id !== $previousLine['c_id']) || ($i === $nbLinesMinus1))) {	//End of current category
+				if ($i === $nbLinesMinus1) {	//End of table
+					$feedsDao[] = $line;
+				}
+				$cat = new Category (
+					$previousLine['c_name'],
+					$previousLine['c_color'],
+					HelperFeed::daoToFeed ($feedsDao)
+				);
+				$cat->_id ($previousLine['c_id']);
+				$list[] = $cat;
+
+				$feedsDao = array();	//Prepare for next category
+				$previousLine = $line;
+				$feedsDao[] = $line;
+			} else {
+				$previousLine = $line;
+				$feedsDao[] = $line;
+			}
+		}
+
+		return $list;
+	}
+
 	public static function daoToCategory ($listDAO) {
 		$list = array ();
 

app/models/EntriesGetter.php

@@ -94,41 +94,53 @@ class EntriesGetter {
 	public function execute () {
 		$entryDAO = new EntryDAO ();
 
-		HelperEntry::$nb = $this->nb;
+		HelperEntry::$nb = $this->nb;	//TODO: Update: Now done in SQL
-		HelperEntry::$first = $this->first;
+		HelperEntry::$first = $this->first;	//TODO: Update: Now done in SQL
 		HelperEntry::$filter = $this->filter;
 
+		$sqlLimit = (empty ($this->filter['words']) && empty ($this->filter['tags'])) ? $this->nb : '';	//Disable SQL LIMIT optimisation during search	//TODO: Do better!
+
 		switch ($this->type['type']) {
 		case 'all':
 			list ($this->entries, $this->next) = $entryDAO->listEntries (
 				$this->state,
-				$this->order
+				$this->order,
+				$this->first,
+				$sqlLimit
 			);
 			break;
 		case 'favoris':
 			list ($this->entries, $this->next) = $entryDAO->listFavorites (
 				$this->state,
-				$this->order
+				$this->order,
+				$this->first,
+				$sqlLimit
 			);
 			break;
 		case 'public':
 			list ($this->entries, $this->next) = $entryDAO->listPublic (
 				$this->state,
-				$this->order
+				$this->order,
+				$this->first,
+				$sqlLimit
 			);
 			break;
 		case 'c':
 			list ($this->entries, $this->next) = $entryDAO->listByCategory (
 				$this->type['id'],
 				$this->state,
-				$this->order
+				$this->order,
+				$this->first,
+				$sqlLimit
 			);
 			break;
 		case 'f':
 			list ($this->entries, $this->next) = $entryDAO->listByFeed (
 				$this->type['id'],
 				$this->state,
-				$this->order
+				$this->order,
+				$this->first,
+				$sqlLimit
 			);
 			break;
 		default:

app/models/Entry.php

@@ -392,12 +392,15 @@ class EntryDAO extends Model_pdo {
 		}
 	}
 
-	public function listWhere ($where, $state, $order, $values = array ()) {
+	private function listWhere ($where, $state, $order, $limitFromId = '', $limitCount = '', $values = array ()) {
 		if ($state == 'not_read') {
 			$where .= ' AND is_read = 0';
 		} elseif ($state == 'read') {
 			$where .= ' AND is_read = 1';
 		}
+		if (!empty($limitFromId)) {	//TODO: Consider using LPAD(e.date, 11)	//CONCAT is for cases when many entries have the same date
+			$where .= ' AND CONCAT(e.date, e.id) ' . ($order === 'low_to_high' ? '<=' : '>=') . ' (SELECT CONCAT(s.date, s.id) FROM ' . $this->prefix . 'entry s WHERE s.id = "' . $limitFromId . '")';
+		}
 
 		if ($order == 'low_to_high') {
 			$order = ' DESC';
@@ -407,78 +410,75 @@ class EntryDAO extends Model_pdo {
 
 		$sql = 'SELECT e.* FROM ' . $this->prefix . 'entry e'
 		     . ' INNER JOIN  ' . $this->prefix . 'feed f ON e.id_feed = f.id' . $where
-		     . ' ORDER BY date' . $order;
+		     . ' ORDER BY e.date' . $order . ', e.id' . $order;
+
+		if (!empty($limitCount)) {
+			$sql .= ' LIMIT ' . ($limitCount + 2);	//TODO: See http://explainextended.com/2009/10/23/mysql-order-by-limit-performance-late-row-lookups/
+		}
+
 		$stm = $this->bd->prepare ($sql);
 		$stm->execute ($values);
 
 		return HelperEntry::daoToEntry ($stm->fetchAll (PDO::FETCH_ASSOC));
 	}
-	public function listEntries ($state, $order = 'high_to_low') {
+	public function listEntries ($state, $order = 'high_to_low', $limitFromId = '', $limitCount = '') {
-		return $this->listWhere (' WHERE priority > 0', $state, $order);
+		return $this->listWhere (' WHERE priority > 0', $state, $order, $limitFromId, $limitCount);
 	}
-	public function listFavorites ($state, $order = 'high_to_low') {
+	public function listFavorites ($state, $order = 'high_to_low', $limitFromId = '', $limitCount = '') {
-		return $this->listWhere (' WHERE is_favorite = 1', $state, $order);
+		return $this->listWhere (' WHERE is_favorite = 1', $state, $order, $limitFromId, $limitCount);
 	}
-	public function listPublic ($state, $order = 'high_to_low') {
+	public function listPublic ($state, $order = 'high_to_low', $limitFromId = '', $limitCount = '') {
-		return $this->listWhere (' WHERE is_public = 1', $state, $order);
+		return $this->listWhere (' WHERE is_public = 1', $state, $order, $limitFromId, $limitCount);
 	}
-	public function listByCategory ($cat, $state, $order = 'high_to_low') {
+	public function listByCategory ($cat, $state, $order = 'high_to_low', $limitFromId = '', $limitCount = '') {
-		return $this->listWhere (' WHERE category = ?', $state, $order, array ($cat));
+		return $this->listWhere (' WHERE category = ?', $state, $order, $limitFromId, $limitCount, array ($cat));
 	}
-	public function listByFeed ($feed, $state, $order = 'high_to_low') {
+	public function listByFeed ($feed, $state, $order = 'high_to_low', $limitFromId = '', $limitCount = '') {
-		return $this->listWhere (' WHERE id_feed = ?', $state, $order, array ($feed));
+		return $this->listWhere (' WHERE id_feed = ?', $state, $order, $limitFromId, $limitCount, array ($feed));
 	}
 
-	public function count () {
+	public function countUnreadRead () {
-		$sql = 'SELECT COUNT(*) AS count FROM ' . $this->prefix . 'entry e INNER JOIN  ' . $this->prefix . 'feed f ON e.id_feed = f.id WHERE priority > 0';
+		$sql = 'SELECT is_read, COUNT(*) AS count FROM ' . $this->prefix . 'entry e INNER JOIN ' . $this->prefix . 'feed f ON e.id_feed = f.id WHERE priority > 0 GROUP BY is_read';
 		$stm = $this->bd->prepare ($sql);
 		$stm->execute ();
 		$res = $stm->fetchAll (PDO::FETCH_ASSOC);
 
-		return $res[0]['count'];
+		$readUnread = array('unread' => 0, 'read' => 0);
+		foreach ($res as $line) {
+			switch (intval($line['is_read'])) {
+				case 0: $readUnread['unread'] = intval($line['count']); break;
+				case 1: $readUnread['read'] = intval($line['count']); break;
 			}
-	public function countNotRead () {
-		$sql = 'SELECT COUNT(*) AS count FROM ' . $this->prefix . 'entry e INNER JOIN  ' . $this->prefix . 'feed f ON e.id_feed = f.id WHERE is_read=0 AND priority > 0';
-		$stm = $this->bd->prepare ($sql);
-		$stm->execute ();
-		$res = $stm->fetchAll (PDO::FETCH_ASSOC);
-
-		return $res[0]['count'];
 		}
-
+		return $readUnread;
-	public function countNotReadByFeed ($id) {
-		$sql = 'SELECT COUNT(*) AS count FROM ' . $this->prefix . 'entry WHERE is_read = 0 AND id_feed = ?';
-		$stm = $this->bd->prepare ($sql);
-		$stm->execute (array ($id));
-		$res = $stm->fetchAll (PDO::FETCH_ASSOC);
-
-		return $res[0]['count'];
 	}
-
+	public function count () {	//Deprecated: use countUnreadRead() instead
-	public function countNotReadByCat ($id) {
+		$unreadRead = $this->countUnreadRead ();	//This makes better use of caching
-		$sql = 'SELECT COUNT(*) AS count FROM ' . $this->prefix . 'entry e INNER JOIN  ' . $this->prefix . 'feed f ON e.id_feed = f.id WHERE is_read=0 AND category = ?';
+		return $unreadRead['unread'] + $unreadRead['read'];
-		$stm = $this->bd->prepare ($sql);
+	}
-		$stm->execute (array ($id));
+	public function countNotRead () {	//Deprecated: use countUnreadRead() instead
-		$res = $stm->fetchAll (PDO::FETCH_ASSOC);
+		$unreadRead = $this->countUnreadRead ();	//This makes better use of caching
-
+		return $unreadRead['unread'];
-		return $res[0]['count'];
 	}
 
-	public function countNotReadFavorites () {
+	public function countUnreadReadFavorites () {
-		$sql = 'SELECT COUNT(*) AS count FROM ' . $this->prefix . 'entry WHERE is_read=0 AND is_favorite=1';
+		$sql = 'SELECT is_read, COUNT(*) AS count FROM ' . $this->prefix . 'entry WHERE is_favorite=1 GROUP BY is_read';
 		$stm = $this->bd->prepare ($sql);
 		$stm->execute ();
 		$res = $stm->fetchAll (PDO::FETCH_ASSOC);
-
+		$readUnread = array('unread' => 0, 'read' => 0);
-		return $res[0]['count'];
+		foreach ($res as $line) {
+			switch (intval($line['is_read'])) {
+				case 0: $readUnread['unread'] = intval($line['count']); break;
+				case 1: $readUnread['read'] = intval($line['count']); break;
+			}
+		}
+		return $readUnread;
 	}
-	public function countFavorites () {
-		$sql = 'SELECT COUNT(*) AS count FROM ' . $this->prefix . 'entry WHERE is_favorite=1';
-		$stm = $this->bd->prepare ($sql);
-		$stm->execute ();
-		$res = $stm->fetchAll (PDO::FETCH_ASSOC);
 
-		return $res[0]['count'];
+	public function countFavorites () {	//Deprecated: use countUnreadReadFavorites() instead
+		$unreadRead = $this->countUnreadReadFavorites ();	//This makes better use of caching
+		return $unreadRead['unread'] + $unreadRead['read'];
 	}
 
 	public function optimizeTable() {
@@ -522,7 +522,7 @@ class HelperEntry {
 					$list[$key] = self::createEntry ($dao);
 
 					$count++;
-					$first_is_found = true;
+					$first_is_found = true;	//TODO: Update: Now done in SQL
 				}
 				if ($count >= self::$nb) {
 					$break_after = true;

app/models/Feed.php

@@ -4,6 +4,7 @@ class Feed extends Model {
 	private $id = null;
 	private $url;
 	private $category = '000000';
+	private $nbNotRead = -1;
 	private $entries = null;
 	private $name = '';
 	private $website = '';
@@ -82,8 +83,12 @@ class Feed extends Model {
 		return $feedDAO->countEntries ($this->id ());
 	}
 	public function nbNotRead () {
+		if ($this->nbNotRead < 0) {
 		$feedDAO = new FeedDAO ();
-		return $feedDAO->countNotRead ($this->id ());
+			$this->nbNotRead = $feedDAO->countNotRead ($this->id ());
+		}
+
+		return $this->nbNotRead;
 	}
 	public function favicon () {
 		$file = '/data/favicons/' . $this->id () . '.ico';
@@ -162,6 +167,13 @@ class Feed extends Model {
 		}
 		$this->keep_history = $value;
 	}
+	public function _nbNotRead ($value) {
+		if (!is_int ($value)) {
+			$value = -1;
+		}
+
+		$this->nbNotRead = intval ($value);
+	}
 
 	public function load () {
 		if (!is_null ($this->url)) {
@@ -493,6 +505,9 @@ class HelperFeed {
 		}
 
 		foreach ($listDAO as $key => $dao) {
+			if (empty ($dao['url'])) {
+				continue;
+			}
 			if (isset ($dao['id'])) {
 				$key = $dao['id'];
 			}
@@ -508,7 +523,9 @@ class HelperFeed {
 			$list[$key]->_httpAuth (base64_decode ($dao['httpAuth']));
 			$list[$key]->_error ($dao['error']);
 			$list[$key]->_keepHistory ($dao['keep_history']);
-
+			if (isset ($dao['nbNotRead'])) {
+				$list[$key]->_nbNotRead ($dao['nbNotRead']);
+			}
 			if (isset ($dao['id'])) {
 				$list[$key]->_id ($dao['id']);
 			}

app/views/helpers/view/normal_view.phtml

@@ -53,7 +53,10 @@ if (isset ($this->entryPaginator) && !$this->entryPaginator->isEmpty ()) {
 				<?php } ?>
 			</li>
 			<?php } ?>
-			<?php $feed = $item->feed (true); ?>
+			<?php
+				$feed = HelperCategory::findFeed($this->cat_aside, $item->feed ());	//We most likely already have the feed object in cache
+				if (empty($feed)) $feed = $item->feed (true);
+			?>
 			<li class="item website"><a href="<?php echo _url ('index', 'index', 'get', 'f_' . $feed->id ()); ?>"><img class="favicon" src="<?php echo $feed->favicon (); ?>" alt="" /> <span><?php echo $feed->name (); ?></span></a></li>
 			<li class="item title"><a target="_blank" href="<?php echo $item->link (); ?>"><?php echo $item->title (); ?></a></li>
 			<li class="item date"><?php echo $item->date (); ?></li>

app/views/helpers/view/reader_view.phtml

@@ -11,7 +11,10 @@ if (isset ($this->entryPaginator) && !$this->entryPaginator->isEmpty ()) {
 	<div class="flux<?php echo !$item->isRead () ? ' not_read' : ''; ?><?php echo $item->isFavorite () ? ' favorite' : ''; ?>" id="flux_<?php echo $item->id (); ?>">
 		<div class="flux_content">
 			<div class="content">
-				<?php $feed = $item->feed (true); ?>
+				<?php
+					$feed = HelperCategory::findFeed($this->cat_aside, $item->feed ());	//We most likely already have the feed object in cache
+					if (empty($feed)) $feed = $item->feed (true);
+				?>
 				<a href="<?php echo $item->link (); ?>">
 					<img class="favicon" src="<?php echo $feed->favicon (); ?>" alt="" /> <span><?php echo $feed->name (); ?></span>
 				</a>