Changeset 1186

Timestamp:

06/26/10 20:41:41 (3 years ago)

Author:

d0nut

Message:

solr crawler finished, solr status page finished, need some kind of automatism

Location:

solr

Files:

• acptemplates/solr.tpl (modified) (4 diffs)
• files/lib/acp/form/SolrForm.class.php (modified) (3 diffs)
• files/lib/data/solr/SolrBridge.php (modified) (10 diffs)

solr/acptemplates/solr.tpl

@@ -20 +20 @@
 .statBarLabel {
         margin-left: 410px;
-}                       
+}
 /*]]>*/
 </style>
 {/capture}
 {include file='header'}
-        
+
 <div class="mainHeadline">
-        <img src="{@RELATIVE_WCF_DIR}icon/statsL.png" alt="" />
+        <img src="{@RELATIVE_WCF_DIR}icon/solrL.png" alt="" />
         <div class="headlineContainer">
-                <h2>{lang}wcf.acp.stats{/lang}</h2>
+                <h2>{lang}wcf.acp.solr{/lang}</h2>
         </div>
 </div>
@@ -38 +38 @@
 
 {if $results|isset && !$results|count}
-        <p class="error">{lang}wcf.acp.stats.noResults{/lang}</p>
+        <p class="error">{lang}wcf.acp.solr.noResults{/lang}</p>
 {/if}
 {*
-<form  method="post" action="index.php?form=Stats">
+<form  method="post" action="index.php?form=Solr">
         <div class="border content">
                 <div class="container-1">
                         <fieldset>
-                                <legend>{lang}wcf.acp.stats.config{/lang}</legend>
-                                
+                                <legend>{lang}wcf.acp.solr.config{/lang}</legend>
+
                                 <div class="formElement{if $errorField == 'username'} formError{/if}">
                                         <div class="formFieldLabel">
@@ -60 +60 @@
                                         </div>
                                 </div>
-                                
+
                                 {if $additionalFields|isset}{@$additionalFields}{/if}
                         </fieldset>
                 </div>
         </div>
-                
+
         <div class="formSubmit">
                 <input type="submit" accesskey="s" value="{lang}wcf.global.button.submit{/lang}" />
@@ -78 +78 @@
                 <div class="container-1">
                         <fieldset>
-                                <legend>{lang}wcf.acp.stats.results{/lang}</legend>
-        
+                                <legend>{lang}wcf.acp.solr.results{/lang}</legend>
+
                                 {foreach from=$results item=result key=$type}
                                         <div class="formElement">
                                                 <p class="formFieldLabel">{$type}</p>
-                                                <div class="formField"><div class="statBar"><div style="width: {$result.percent|round}%;"></div></div><p class="statBarLabel">{#$result.current}/{#$result.total}</p></div>
+                                                <div class="formField">
+                                                        <div class="statBar"><div style="width: {$result.percent|round}%;"></div></div>
+                                                        <p class="statBarLabel">
+                                                                {$result.percent|round}% {#$result.current}/{#$result.total}
+                                                        </p>
+                                                </div>
                                         </div>
-                                
+
                                 {/foreach}
                         </fieldset>

solr/files/lib/acp/form/SolrForm.class.php

@@ -26 +26 @@
          */
         public $activeMenuItem = 'wcf.acp.menu.link.solr.index';
-        
-        /**
-         * Permission
-         *
-         * @var string
-         */
-        public $neededPermissions = 'admin.contest.canAddClass';
 
 
@@ -44 +37 @@
                 parent::readFormParameters();
 
-                if (isset($_POST['topic'])) $this->topic = StringUtil::trim($_POST['topic']);
+                if (isset($_POST['type'])) $this->type = StringUtil::trim($_POST['type']);
         }
 
@@ -55 +48 @@
                 $this->bridge = new SolrBridge();
                 $this->status = $this->bridge->getIndexStatus();
-        }
-
-        /**
-         * @see Form::save()
-         */
-        public function save() {
-                parent::save();
-
-                // save
-                $this->contestClass = ContestClassEditor::create($this->topic, $this->text, 
-                        $this->parentClassID, $this->position, WCF::getLanguage()->getLanguageID());
-                $this->saved();
-
-                // reset values
-                $this->topic = $this->text = $this->parentClassID = '';
-                $this->languageID = $this->position = 0;
-
-                // show success message
-                WCF::getTPL()->assign('success', true);
+                
+                # demo call for crawling
+                # $this->bridge->doCrawl(array('lexicon'), 50);
         }
 

solr/files/lib/data/solr/SolrBridge.php

@@ -7 +7 @@
 /**
  *
- */     
+ */
 class SolrBridge {
-        
-        /**
-         * 
+
+        /**
+         *
          * @var array<Apache_Solr_Document>
          */
         protected $documents = array();
-        
+
         /**
          *
@@ -21 +21 @@
          */
         protected $solr = null;
-        
+
+        protected static $typeids = null;
+
         /**
          *
          */
         public function __construct() {
-        
+
                 // load search type objects
                 SearchEngine::getSearchTypes();
-                
+
                 $this->solr = new SolrService();
         }
-        
+
         /**
          *
          */
         protected function commit() {
-                $this->solr->addDocuments( $this->documents );
-                $this->solr->commit();
-                
+                #$this->solr->addDocuments( $this->documents );
+                #$this->solr->commit();
+
                 // mark items as done
                 $sql = "INSERT IGNORE INTO
-                                        wcf".WCF_N."_solr_index
-                                        (typeID, messageID)";
+                                wcf".WCF_N."_solr_index
+                                (typeID, messageID)
+                        VALUES ";
                 foreach($this->documents as $doc) {
-                        $sql .= ""; //TODO: mark as done
-                }
+                        $typeID = $this->getTypeID($doc->messageType);
+                        $sql .= "($typeID,".$doc->messageID."),";
+                }
+                $sql = rtrim($sql, ',');
                 $result = WCF::getDB()->sendQuery($sql);
-                
+
                 // reset array
                 $this->documents = array();
-                
-                $this->solr->optimize();
-        }
-        
+
+                // optimize solr index
+                #$this->solr->optimize();
+        }
+
         /**
          *
@@ -67 +73 @@
                         if (!$doc->isAccessible()) continue;
                         if (!empty($sql)) $sql .= "\nUNION\n";
-                        
+
                         // get field names
                         $messageIDFieldName = $doc->getIDFieldName();
                         $messageIDFieldName = strpos($messageIDFieldName, '.') !== false ? $messageIDFieldName : "messageTable.".$messageIDFieldName;
-                        
-                        $sql .= "(      
+
+                        $sql .= "(
                                 SELECT          ".$func."(".$messageIDFieldName.") AS messageID,
                                                 '".$type."' AS messageType
@@ -79 +85 @@
                         )";
                 }
-                
+
                 // send search query
                 $types = array();
@@ -86 +92 @@
                         $types[$row['messageType']] = $row['messageID'];
                 }
-                
+
                 return $types;
         }
-        
-        /**
-         *
-         */
-        public function loadDocuments($type, $min, $max) {
+
+        private function cleanText($message) {
+                require_once(WCF_DIR.'lib/data/message/bbcode/MessageParser.class.php');
+
+                // add cache resources
+                WCF::getCache()->addResource('bbcodes', WCF_DIR.'cache/cache.bbcodes.php', WCF_DIR.'lib/system/cache/CacheBuilderBBCodes.class.php');
+                WCF::getCache()->addResource('smileys', WCF_DIR.'cache/cache.smileys.php', WCF_DIR.'lib/system/cache/CacheBuilderSmileys.class.php');
+
+                $parser = MessageParser::getInstance();
+                $parser->setOutputType('text/plain');
+                $message = StringUtil::stripHTML($message);
+                return $parser->parse($message, false, false, true, false);
+        }
+
+        /**
+         *
+         * @return      integer number of added documents
+         */
+        public function loadDocuments($type, $min, $max, $limit) {
                 // get search type object
                 $doc = SearchEngine::$searchTypeObjects[$type];
                 if (!$doc->isAccessible()) continue;
-                
+
                 // get field names
                 $messageIDFieldName = $doc->getIDFieldName();
@@ -106 +126 @@
                 $usernameFieldName = $doc->getUsernameFieldName();
                 $timeFieldName = $doc->getTimeFieldName();
-                
-                $sql = "SELECT          
+
+                $sql = "SELECT
                                         '".$type."' AS messageType,
                                         ".$messageIDFieldName." AS messageID,
@@ -118 +138 @@
                                         ".$doc->getJoins()."
                         WHERE           ".$messageIDFieldName." BETWEEN $min AND $max
-                                        ".(!empty($conditions[$type]) ? " ".(!empty($q) ? "AND" : "")." (".$conditions[$type].")" : "")."
-                        GROUP BY        messageID";
+                        GROUP BY        messageID
+                        ORDER BY        messageID ASC";
 
                 $result = WCF::getDB()->sendQuery($sql, $limit);
+                $i = 0;
                 while ($row = WCF::getDB()->fetchArray($result)) {
+                        $row['message'] = $this->cleanText($row['message']);
                         $this->addDocument($row);
-                }
-        }
-        
+                        $i++;
+                }
+                return $i;
+        }
+
         /**
          *
@@ -142 +166 @@
                         }
                 }
-                
+
                 $this->documents[] = $part;
         }
-        
-        /**
-         *
-         */
-        public function doCrawl($types, $limit) {
-                foreach($this->getIndexStatus($types) as $typeName => $status) {
-                
+
+        /**
+         *
+         */
+        public function doCrawl($types = null, $limit = null) {
+
+                // get types
+                $types = is_array($types) ? $types : $this->getSearchTypes();
+
+                $i = 0;
+
+                foreach($this->getIndexStatus($types, 'MAX') as $type => $status) {
+
                         // nothing to do?
-                        if($row['total'] == $row['current']) {
+                        if($status['total'] == $status['current']) {
                                 continue;
                         }
-                        
+                // get search type object
+                $doc = SearchEngine::$searchTypeObjects[$type];
+                if (!$doc->isAccessible()) continue;
+
                         if (!isset(SearchEngine::$searchTypeObjects[$type])) {
                                 throw new SystemException('unknown search type '.$type, 101001);
                         }
+                        $j = $this->loadDocuments($type, $status['current'] + 1, $status['total'], $limit);
+                        if($j) {
+                                $i += $j;
+
+                                // write to solr
+                                $this->commit();
+                        }
+                }
+
+                return $i;
+        }
+
+        private function getTypeID($type) {
+
+                if(self::$typeids === null) {
+                        self::$typeids = array();
+
+                        $sql = 'SELECT          *
+                                FROM            wcf'.WCF_N.'_searchable_message_type';
+                        $result = WCF::getDB()->sendQuery($sql);
+                        while ($row = WCF::getDB()->fetchArray($result)) {
+                                self::$typeids[$row['typeName']] = $row['typeID'];
+                        }
+                }
+                return self::$typeids[$type];
+        }
+        
+        private function getSearchTypes() {
+                $types = SearchEngine::getSearchTypes();
+                $return = array();
+                foreach($types as $type) {
+                        $doc = SearchEngine::$searchTypeObjects[$type];
+                        if (!$doc->isAccessible()) continue;
                         
-                        $this->loadDocuments($row['current'] + 1, min($row['total'], $row['current'] + 1 + $limit));
-
-                        // write to solr
-                        $this->commit();
-                }
-        }
-        
-        /**
-         * 
-         */
-        public function getIndexStatus($types = null) {
-                
+                        $return[] = $type;
+                }
+                return $return;
+        }
+
+        /**
+         *
+         */
+        public function getIndexStatus($types = null, $func = 'COUNT') {
+
                 // read available types
                 $status = array();
-                
+
                 // get types
-                $types = is_array($types) ? $types : SearchEngine::getSearchTypes();
-                
+                $types = is_array($types) ? $types : $this->getSearchTypes();
+
                 // set counters to zero
                 foreach ($types as $type) {
@@ -187 +251 @@
                         );
                 }
-                
+
                 // read current status
                 $sql = 'SELECT          typeName,
                                         c
                         FROM (
-                                SELECT          typeID, 
-                                                COUNT(typeID) AS c
-                                FROM            wcf'.WCF_N.'_solr_index 
+                                SELECT          typeID,
+                                                '.$func.'(messageID) AS c
+                                FROM            wcf'.WCF_N.'_solr_index
                                 GROUP BY        typeID
                         ) x
-                        INNER JOIN      wcf'.WCF_N.'_searchable_message_type USING(typeID)';
-                while ($row = WCF::getDB()->fetchArray($sql)) {
+                        INNER JOIN      wcf'.WCF_N.'_searchable_message_type USING(typeID)
+                        WHERE           typeName IN ("'.implode('","', $types).'")';
+                $result = WCF::getDB()->sendQuery($sql);
+                while ($row = WCF::getDB()->fetchArray($result)) {
                         $typeName = $row['typeName'];
                         $status[$typeName]['current'] = $row['c'];
@@ -204 +270 @@
 
                 // read totals
-                foreach ($this->getTotals($types, 'COUNT') as $typeName => $count) {
-                        $percent = $count ? 100 / $count * $status[$typeName]['current'] : 0;
+                foreach ($this->getTotals($types, $func) as $typeName => $count) {
                         $status[$typeName]['total'] = $count;
-                }
-                
+                        $status[$typeName]['percent'] = $count ? 100 / $count * $status[$typeName]['current'] : 0;
+                }
+
                 return $status;
         }