Major refactoring: add a cache table to accelerate the search of linked tags by inference (children and synonyms)

git-svn-id: https://semanticscuttle.svn.sourceforge.net/svnroot/semanticscuttle/trunk@204 b3834d28-1941-0410-a4f8-b48e95affb8f

admin.php

@@ -23,6 +23,7 @@ require_once('header.inc.php');
 $bookmark2tagservice = & ServiceFactory :: getServiceInstance('Bookmark2Tagservice');
 $bookmarkservice = & ServiceFactory :: getServiceInstance('BookmarkService');
 $tag2tagservice = & ServiceFactory :: getServiceInstance('Tag2TagService');
+$tagcacheservice = & ServiceFactory :: getServiceInstance('TagCacheService');
 
 // Header variables
 $tplVars['subtitle'] = T_('Manage users');
@@ -52,6 +53,7 @@ if ( $action
 			if ( $user && ($userinfo = $userservice->getUserByUsername($user)) ) {
 				$uId = $userinfo['uId'];
 
+				$tagcacheservice->deleteByUser($uId);
 				$tag2tagservice->removeLinkedTags('','','',$uId);
 				$userservice->deleteUser($uId);
 				$bookmark2tagservice->deleteTagsForUser($uId);				

index.php

@@ -21,7 +21,6 @@
 
 require_once('header.inc.php');
 
-
 /* Service creation: only useful services are created */
 $bookmarkservice =& ServiceFactory::getServiceInstance('BookmarkService');
 $cacheservice =& ServiceFactory::getServiceInstance('CacheService');

services/bookmark2tagservice.php

@@ -25,6 +25,9 @@ class Bookmark2TagService {
 	function attachTags($bookmarkid, $tags, $fromApi = false, $extension = NULL, $replace = true, $fromImport = false) {
 		// Make sure that categories is an array of trimmed strings, and that if the categories are
 		// coming in from an API call to add a bookmark, that underscores are converted into strings.
+		
+		trigger_error($GLOBALS['filetypes'].'bbbbbbbbbbbbbbbbbbbbbbbbbbbb');
+		
 		if (!is_array($tags)) {
 			$tags = trim($tags);
 			if ($tags != '') {
@@ -70,6 +73,7 @@ class Bookmark2TagService {
 		// Media and file types
 		if (!is_null($extension)) {
 			include_once(dirname(__FILE__) .'/../functions.inc.php');
+			
 			if ($keys = multi_array_search($extension, $GLOBALS['filetypes'])) {
 				$tags[] = 'system:filetype:'. $extension;
 				$tags[] = 'system:media:'. array_shift($keys);

services/bookmarkservice.php

@@ -154,6 +154,8 @@ class BookmarkService {
 		$extension = end($uriparts);
 		unset($uriparts);
 		
+		trigger_error($GLOBALS['filetypes'].'aaaaaaaaaaaaaaaaaaaaa');
+
 		$b2tservice = & ServiceFactory :: getServiceInstance('Bookmark2TagService');
 		if (!$b2tservice->attachTags($bId, $categories, $fromApi, $extension, false, $fromImport)) {
 			$this->db->sql_transaction('rollback');

services/servicefactory.php

@@ -17,6 +17,7 @@ class ServiceFactory {
 				message_die(CRITICAL_ERROR, "Could not connect to the database", $db);
 			}
 		}		
+		
 		if (!isset($instances[$name])) {
 			if (isset($serviceoverrules[$name])) {
 				$name = $serviceoverrules[$name];
@@ -25,6 +26,7 @@ class ServiceFactory {
 				if (!isset($servicedir)) {
 					$servicedir = dirname(__FILE__) .'/';
 				}
+								
 				require_once($servicedir . strtolower($name) . '.php');
 			}
 			$instances[$name] = call_user_func(array($name, 'getInstance'), $db);

services/tag2tagservice.php

@@ -37,9 +37,8 @@ class Tag2TagService {
 		}
 		$this->db->sql_transaction('commit');
 
-		// Update stats
-		$tsts =& ServiceFactory::getServiceInstance('TagStatService');
-		$tsts->updateStat($tag1, $relationType, $uId);
+		// Update stats and cache
+		$this->update($tag1, $tag2, $relationType, $uId);
 
 		return true;
 	}
@@ -77,7 +76,6 @@ class Tag2TagService {
 		$output = array();
 		foreach($rowset as $row) {
 			if(!in_array($row['tag'], $stopList)) {
-
 				$output[] = $row['tag'];
 			}
 		}
@@ -93,49 +91,69 @@ class Tag2TagService {
 		return $output;
 	}
 
-	/* TODO: clean the outputs to obtain homogenous ones*/
-	function getAllLinkedTags($tag1, $relationType, $uId, $asFlatList=true, $stopList=array()) {
-		$asFlatList = true; //we disable the tree list parameter for the moment
-
+	/*
+	 * Returns all linked tags (all descendants if relation is >,
+	 * all synonyms if relation is = )
+	 * $stopList allows to avoid cycle (a > b > a) between tags
+	 */
+	function getAllLinkedTags($tag1, $relationType, $uId, $stopList=array()) {
 		if(in_array($tag1, $stopList) || $tag1 == '') {
 			return array();
 		}
 
+		// try to find data in cache
+		$tcs = & ServiceFactory::getServiceInstance('TagCacheService');
+		if(count($stopList) == 0) {
+			$activatedCache = true;
+		} else {
+			$activatedCache = false;
+		}
+
+		// look for existing links
 		$stopList[] = $tag1;
 		$linkedTags = $this->getLinkedTags($tag1, $relationType, $uId, false, $stopList);
-
 		if($relationType != '=') {
 			$linkedTags = array_merge($linkedTags, $this->getLinkedTags($tag1, '=', $uId, false, $stopList));
 		}
 
 		if(count($linkedTags) == 0) {
 			return array();
+
 		} else {
-			$output = array();
-			if($asFlatList == true) {
-				//$output[$tag1] = $tag1;
-			} else {
-				$output = array('node'=>$tag1);
+			// use cache if possible
+			if($activatedCache) {
+				if($relationType == '>') {
+					$output = $tcs->getChildren($tag1, $uId);
+				} elseif($relationType == '=') {
+					$output = $tcs->getSynonyms($tag1, $uId);
+				}
+				if(count($output)>0) {
+					return $output;
+				}
 			}
 
-			foreach($linkedTags as $linkedTag) {
-				$allLinkedTags = $this->getAllLinkedTags($linkedTag, $relationType, $uId, $asFlatList, $stopList);
+			// else compute the links
+			$output = array();
 
-				if($asFlatList == true) {
+			foreach($linkedTags as $linkedTag) {
+				$allLinkedTags = $this->getAllLinkedTags($linkedTag, $relationType, $uId, $stopList);
 				$output[] = $linkedTag;
 				if(is_array($allLinkedTags)) {
-							
 					$output = array_merge($output, $allLinkedTags);
 				} else {
 					$output[] = $allLinkedTags;
 				}
-				} else {
-					$output[] = $allLinkedTags;
-				}
 			}
+
+			// and save in cache
+			if($activatedCache == true) {
+				$tcs->updateTag($tag1, $relationType, $output, $uId);
 			}
+				
 			//$output = array_unique($output); // remove duplication
 			return $output;
+
+		}
 	}
 
 	function getOrphewTags($relationType, $uId = 0, $limit = null, $orderBy = null) {
@@ -263,9 +281,9 @@ class Tag2TagService {
 			return false;
 		}
 
-		// Update stats
-		$tsts =& ServiceFactory::getServiceInstance('TagStatService');
-		$tsts->updateStat($tag1, $relationType, $uId);
+
+		// Update stats and cache
+		$this->update($tag1, $tag2, $relationType, $uId);
 
 		return true;
 	}
@@ -286,17 +304,25 @@ class Tag2TagService {
 		$query.= ' AND uId="'.$uId.'"';
 		$this->db->sql_query($query);
 
-		// Update stats
-		$tsts =& ServiceFactory::getServiceInstance('TagStatService');
-		$tsts->updateStat($oldName, '=', $uId);
-		$tsts->updateStat($oldName, '>', $uId);
-		$tsts->updateStat($newName, '=', $uId);
-		$tsts->updateStat($newName, '>', $uId);
+
+		// Update stats and cache
+		$this->update($oldName, NULL, '=', $uId);
+		$this->update($oldName, NULL, '>', $uId);
+		$this->update($newName, NULL, '=', $uId);
+		$this->update($newName, NULL, '>', $uId);
 
 		return true;
 
 	}
 
+	function update($tag1, $tag2, $relationType, $uId) {
+		$tsts =& ServiceFactory::getServiceInstance('TagStatService');
+		$tsts->updateStat($tag1, $relationType, $uId);
+
+		$tcs = & ServiceFactory::getServiceInstance('TagCacheService');
+		$tcs->deleteByUser($uId);
+	}
+
 	function deleteAll() {
 		$query = 'TRUNCATE TABLE `'. $this->getTableName() .'`';
 		$this->db->sql_query($query);

services/tagcacheservice.php

@@ -0,0 +1,345 @@
+<?php
+
+/*
+ * This class infers on relation between tags by storing all the including tags or synonymous tag.
+ * For example, if the user creates: tag1>tag2>tag3, the system can infer that tag is included into tag1.
+ * Instead of computing this relation several times, it is saved into this current table.
+ * For synonymy, this table stores also the group of synonymous tags.
+ * The table must be updated for each modification of the relations between tags.
+ */
+
+class TagCacheService {
+	var $db;
+	var $tablename;
+
+	function &getInstance(&$db) {
+		static $instance;
+		if (!isset($instance))
+		$instance =& new TagCacheService($db);
+		return $instance;
+	}
+
+	function TagCacheService(&$db) {
+		$this->db =& $db;
+		$this->tablename = $GLOBALS['tableprefix'] .'tagscache';
+	}
+
+	function getChildren($tag1, $uId) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag1 = $tagservice->normalize($tag1);
+
+		if($tag1 == '') return false;
+
+		$query = "SELECT DISTINCT tag2 as 'tag'";
+		$query.= " FROM `". $this->getTableName() ."`";
+		$query.= " WHERE relationType = '>'";
+		$query.= " AND tag1 = '".$tag1."'";
+		$query.= " AND uId = '".$uId."'";
+
+		//die($query);
+		if (! ($dbresult =& $this->db->sql_query($query)) ){
+			message_die(GENERAL_ERROR, 'Could not get related tags', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+
+
+		$rowset = $this->db->sql_fetchrowset($dbresult);
+		$output = array();
+		foreach($rowset as $row) {
+			$output[] = $row['tag'];
+		}
+
+		return $output;
+	}
+
+	function addChild($tag1, $tag2, $uId) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag1 = $tagservice->normalize($tag1);
+		$tag2 = $tagservice->normalize($tag2);
+
+		if($tag1 == $tag2 || strlen($tag1) == 0 || strlen($tag2) == 0
+		|| ($this->existsChild($tag1, $tag2, $uId))) {
+			return false;
+		}
+
+		$values = array('tag1' => $tag1, 'tag2' => $tag2, 'relationType'=> '>', 'uId'=> $uId);
+		$query = 'INSERT INTO '. $this->getTableName() .' '. $this->db->sql_build_array('INSERT', $values);
+		//die($query);
+		if (!($dbresult =& $this->db->sql_query($query))) {
+			$this->db->sql_transaction('rollback');
+			message_die(GENERAL_ERROR, 'Could not add tag cache inference', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+		$this->db->sql_transaction('commit');
+	}
+
+	function removeChild($tag1, $tag2, $uId) {
+		if(($tag1 != '' && $tag1 == $tag2) ||
+		($tag1 == '' && $tag2 == '' && $uId == '')) {
+			return false;
+		}
+
+		$query = 'DELETE FROM '. $this->getTableName();
+		$query.= ' WHERE 1=1';
+		$query.= strlen($tag1)>0 ? ' AND tag1 = "'. $tag1 .'"' : '';
+		$query.= strlen($tag2)>0 ? ' AND tag2 = "'. $tag2 .'"' : '';
+		$query.= ' AND relationType = ">"';
+		$query.= strlen($uId)>0 ? ' AND uId = "'. $uId .'"' : '';
+
+		if (!($dbresult =& $this->db->sql_query($query))) {
+			message_die(GENERAL_ERROR, 'Could not remove tag cache inference', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+	}
+	
+	function removeChildren($tag1, $uId) {
+		$this->removeChild($tag1, NULL, $uId);
+	}
+
+	function existsChild($tag1, $tag2, $uId) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag1 = $tagservice->normalize($tag1);
+		$tag2 = $tagservice->normalize($tag2);
+
+		$query = "SELECT tag1, tag2, relationType, uId FROM `". $this->getTableName() ."`";
+		$query.= " WHERE tag1 = '" .$tag1 ."'";
+		$query.= " AND tag2 = '".$tag2."'";
+		$query.= " AND relationType = '>'";
+		$query.= " AND uId = '".$uId."'";
+
+		//echo($query."<br>\n");
+
+		return $this->db->sql_numrows($this->db->sql_query($query)) > 0;
+
+	}
+
+	/*
+	 * Synonyms of a same concept are a group. A group has one main synonym called key
+	 * and a list of synonyms called values.
+	 */
+	function addSynonym($tag1, $tag2, $uId) {
+
+		if($tag1 == $tag2 || strlen($tag1) == 0 || strlen($tag2) == 0
+		|| ($this->existsSynonym($tag1, $tag2, $uId))) {
+			return false;
+		}
+
+		$case1 = '0'; // not in DB
+		if($this->_isSynonymKey($tag1, $uId)) {
+			$case1 = 'key';
+		} elseif($this->_isSynonymValue($tag1, $uId)) {
+			$case1 = 'value';
+		}
+
+		$case2 = '0'; // not in DB
+		if($this->_isSynonymKey($tag2, $uId)) {
+			$case2 = 'key';
+		} elseif($this->_isSynonymValue($tag2, $uId)) {
+			$case2 = 'value';
+		}
+		$case = $case1.$case2;
+
+		// all the possible cases
+		switch ($case) {
+			case 'keykey':
+				$values = $this->_getSynonymValues($tag2, $uId);
+				$this->removeSynonymGroup($tag2, $uId);
+				foreach($values as $value) {
+					$this->addSynonym($tag1, $value['tag'], $uId);
+				}
+				$this->addSynonym($tag1, $tag2, $uId);
+				break;
+
+			case 'valuekey':
+				$key = $this->_getSynonymKey($tag1, $uId);
+				$this->addSynonym($key, $tag2, $uId);
+				break;
+
+			case 'keyvalue':
+				$this->addSynonym($tag2, $tag1, $uId);
+				break;
+			case 'valuevalue':
+				$key1 =  $this->_getSynonymKey($tag1, $uId);
+				$key2 =  $this->_getSynonymKey($tag2, $uId);
+				$this->addSynonym($key1, $key2, $uId);
+				break;
+			case '0value':
+				$key = $this->_getSynonymKey($tag2, $uId);
+				$this->addSynonym($key, $tag1, $uId);
+				break;
+			case 'value0':
+				$this->addSynonym($tag2, $tag1, $uId);
+				break;
+			case '0key':
+				$this->addSynonym($tag2, $tag1, $uId);
+				break;
+			default:
+				$values = array('tag1' => $tag1, 'tag2' => $tag2, 'relationType'=> '=', 'uId'=> $uId);
+				$query = 'INSERT INTO '. $this->getTableName() .' '. $this->db->sql_build_array('INSERT', $values);
+				//die($query);
+				if (!($dbresult =& $this->db->sql_query($query))) {
+					$this->db->sql_transaction('rollback');
+					message_die(GENERAL_ERROR, 'Could not add tag cache synonymy', '', __LINE__, __FILE__, $query, $this->db);
+					return false;
+				}
+				$this->db->sql_transaction('commit');
+				break;
+		}
+	}
+
+	function removeSynonymGroup($tag1, $uId) {		
+		$query = 'DELETE FROM '. $this->getTableName();
+		$query.= ' WHERE 1=1';
+		$query.= ' AND tag1 = "'. $tag1 .'"';
+		$query.= ' AND relationType = "="';
+		$query.= ' AND uId = "'. $uId .'"';
+
+		if (!($dbresult =& $this->db->sql_query($query))) {
+			message_die(GENERAL_ERROR, 'Could not remove tag cache inference', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+	}
+
+	function _isSynonymKey($tag1, $uId) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag1 = $tagservice->normalize($tag1);
+
+		$query = "SELECT tag1 FROM `". $this->getTableName() ."`";
+		$query.= " WHERE tag1 = '" .$tag1 ."'";
+		$query.= " AND relationType = '='";
+		$query.= " AND uId = '".$uId."'";
+
+		return $this->db->sql_numrows($this->db->sql_query($query)) > 0;
+	}
+
+	function _isSynonymValue($tag2, $uId) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag2 = $tagservice->normalize($tag2);
+
+		$query = "SELECT tag2 FROM `". $this->getTableName() ."`";
+		$query.= " WHERE tag2 = '" .$tag2 ."'";
+		$query.= " AND relationType = '='";
+		$query.= " AND uId = '".$uId."'";
+
+		return $this->db->sql_numrows($this->db->sql_query($query)) > 0;
+	}
+
+	function getSynonyms($tag1, $uId) {	
+		$values = array();	
+		if($this->_isSynonymKey($tag1, $uId)) {
+			$values = $this->_getSynonymValues($tag1, $uId);
+		} elseif($this->_isSynonymValue($tag1, $uId)) {
+			$key = $this->_getSynonymKey($tag1, $uId);
+			$values = $this->_getSynonymValues($key, $uId, $tag1);
+			$values[] = $key;			
+		}
+		return $values;
+	}
+
+	function _getSynonymKey($tag2, $uId) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag2 = $tagservice->normalize($tag2);
+
+		if($this->_isSynonymKey($tag2)) return $tag2;
+		
+		if($tag2 == '') return false;
+
+		$query = "SELECT DISTINCT tag1 as 'tag'";
+		$query.= " FROM `". $this->getTableName() ."`";
+		$query.= " WHERE relationType = '='";
+		$query.= " AND tag2 = '".$tag2."'";
+		$query.= " AND uId = '".$uId."'";
+
+		//die($query);
+		if (! ($dbresult =& $this->db->sql_query($query)) ){
+			message_die(GENERAL_ERROR, 'Could not get related tags', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+
+		$row = $this->db->sql_fetchrow($dbresult);
+		return $row['tag'];
+	}
+
+	/*
+	 * Return values associated with a key.
+	 * $tagExcepted allows to hide a value.
+	 */
+	function _getSynonymValues($tag1, $uId, $tagExcepted = NULL) {
+		$tagservice =& ServiceFactory::getServiceInstance('TagService');
+		$tag1 = $tagservice->normalize($tag1);
+		$tagExcepted = $tagservice->normalize($tagExcepted);
+
+		if($tag1 == '') return false;
+
+		$query = "SELECT DISTINCT tag2 as 'tag'";
+		$query.= " FROM `". $this->getTableName() ."`";
+		$query.= " WHERE relationType = '='";
+		$query.= " AND tag1 = '".$tag1."'";
+		$query.= " AND uId = '".$uId."'";
+		$query.= $tagExcepted!=''?" AND tag2!='".$tagExcepted."'":"";
+
+		if (! ($dbresult =& $this->db->sql_query($query)) ){
+			message_die(GENERAL_ERROR, 'Could not get related tags', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+
+		$rowset = $this->db->sql_fetchrowset($dbresult);
+
+		$output = array();
+		foreach($rowset as $row) {
+			$output[] = $row['tag'];
+		}
+		return $output;
+	}
+
+	function existsSynonym($tag1, $tag2, $uId) {
+		if($this->_getSynonymKey($tag1, $uId) == $tag2 || $this->_getSynonymKey($tag2, $uId) == $tag1) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+
+	function updateTag($tag1, $relationType, $otherTags, $uId) {
+		if($relationType == '=') {
+			if($this->getSynonyms($tag1, $uId)) {  // remove previous data avoiding unconstistency
+				$this->removeSynonymGroup($tag1, $uId);
+			}
+				
+			foreach($otherTags as $tag2) {
+				$this->addSynonym($tag1, $tag2, $uId);
+			}
+		} elseif($relationType == '>') {
+			if(count($this->getChildren($tag1, $uId))>0) { // remove previous data avoiding unconstistency
+				$this->removeChildren($tag1);
+			}
+			
+			foreach($otherTags as $tag2) {
+				$this->addChild($tag1, $tag2, $uId);
+			}
+		}		
+	}
+
+	function deleteByUser($uId) {
+		$query = 'DELETE FROM '. $this->getTableName() .' WHERE uId = '. intval($uId);
+
+		if (!($dbresult = & $this->db->sql_query($query))) {
+			message_die(GENERAL_ERROR, 'Could not delete user tags cache', '', __LINE__, __FILE__, $query, $this->db);
+			return false;
+		}
+
+		return true;
+
+	}
+
+	function deleteAll() {
+		$query = 'TRUNCATE TABLE `'. $this->getTableName() .'`';
+		$this->db->sql_query($query);
+	}
+
+	// Properties
+	function getTableName()       { return $this->tablename; }
+	function setTableName($value) { $this->tablename = $value; }
+}
+?>

services/tagstatservice.php

@@ -117,7 +117,7 @@ class TagStatService {
 		$query = "SELECT tag1, uId FROM `". $tts->getTableName() ."`";
 		$query.= " WHERE relationType = '>'";
 
-//die($query);
+		//die($query);
 
 		if (! ($dbresult =& $this->db->sql_query($query)) ){
 			message_die(GENERAL_ERROR, 'Could not update stats', '', __LINE__, __FILE__, $query, $this->db);

tables.sql

@@ -121,6 +121,22 @@ CREATE TABLE `sc_tagsstats` (
 
 -- --------------------------------------------------------
 
+-- 
+-- Table structure for table `sc_tagscache`
+-- 
+
+CREATE TABLE `sc_tagscache` (
+  `tcId` int(11) NOT NULL auto_increment,
+  `tag1` varchar(100) NOT NULL default '',
+  `tag2` varchar(100) NOT NULL default '',
+  `relationType` varchar(32) NOT NULL default '',
+  `uId` int(11) NOT NULL default '0',
+  PRIMARY KEY (`tcId`),
+  UNIQUE KEY `sc_tagscache_tag1_tag2_type_uId` (`tag1`,`tag2`,`relationType`,`uId`)
+);
+
+-- --------------------------------------------------------
+
 -- 
 -- Table structure for table `sc_commondescription`
 --