Ik ben hier ook eens een tijdje mee bezig geweest en
postte destijds ook al hierover. Ik ben altijd van plan geweest er een mooie tutorial van te gaan maken maar dat is er nooit van gekomen. Wel heb ik een
demootje online. Die errored alleen bij geen resultaten en hij cached heel smerig alle output, dus zoeken op 'god' is niet veel meer dan een readfile($cache)
. Maar voor de rest werkt het alleraardigst.
Hier ook nog de relevante sourcecode, de base-classes moet je er zelf maar bij verzinnen
Waar het op neerkomt is dat ik het MYSQL fulltext algoritme heb nagebouwd in PHP, en er alleen net iets meer mee kan. Betekent wel dat je per tabel met zoekbare text een extra veld heb die de fulltext index nadoet (waar dan met LIKE in word gezocht), en nog 2 magische getallen waarvan ik ook niet meer precies weet hoe ze ok al weer werkten, $norm_pivot en $sum_dtf.
[edit]
De demo staat trouwens standaard op AND, en ik zie ook dat als je die via de URL op OR zet, de weging geen extra punten toekend aan AND. Maar dat moest ik idd nog doen
Maar goed. Daar gaat het niet om
. Of ja daar gaat het eigenlijk wel om maar ik heb geen zin om dat nu gaan veranderen
PHP:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
| <?
import("nl.shapers.search.LikeSearch");
function order_objects_by_relevance_desc($a, $b) {
if ($a->relevance == $b->relevance) {
return 0;
}
return ($a->relevance > $b->relevance) ? -1 : 1;
}
class FtLikeSearch extends LikeSearch {
var $variantFactors = array("exact" => 2,
"start" => 1.5,
"end" => 1,
"mid" => 0.5);
var $variants = array();
function FtLikeSearch() {
LikeSearch::LikeSearch();
}
function setStopwords($file) {
global $stopwords;
if(!isset($GLOBALS['stopwords'])) {
$ext = end(explode(".",$file));
if($ext == "php") {
include($file);
}
elseif($ext == "txt") {
$phpfile = preg_replace("/txt$/", "php", $file);
if(file_exists($phpfile)) {
include($phpfile);
}
else {
$contents = file_get_contents($file);
$regexes = "/(\s|^)(".str_replace(",","(\s|$))+/,/(\s|^)(", $contents).")(\s|$)/";
$stopwords = explode("," , $regexes);
FtLikeSearch::compileStopwords($phpfile);
}
}
}
return is_array($GLOBALS['stopwords']);
}
function compileStopwords($file) {
$contents = "<?\n\$stopwords = array(\"".implode("\",\"", $GLOBALS["stopwords"])."\");\n?>";
$fp = fopen($file, "w");
fwrite($fp, $contents);
fclose($fp);
}
function setQuery($query, $stopwordsFile = SEARCH_STOPWORDS_FILE) {
BaseSearch::setQuery($query);
$this->setStopwords($stopwordsFile);
$stoppedQuery = preg_replace($GLOBALS['stopwords'], "", $this->query);
if(strlen($stoppedQuery) > 0) {
$this->words = explode(" ", $stoppedQuery);
foreach($this->words as $n => $word) {
$this->words[$n] = strtolower($word);
$this->variants[$n] = array();
$this->variants[$n]["exact"] = " ".$word." ";
$this->variants[$n]["start"] = " ".$word;
$this->variants[$n]["end"] = $word." ";
$this->variants[$n]["mid"] = $word;
}
$this->setPregWords($this->words);
}
else {
$this->words = null;
}
}
function getResults(& $count, $from = null, $to = null) {
if(SEARCH_DB_ENGINE == "Propel") {
$results = BaseSearch::getResultsPropel();
}
else {
$results = BaseSearch::getResults();
}
if($this->merge) {
//$results = $this->mergeResults($results);
}
foreach($results as $class => $objects) {
$count[$class] = count($objects);
if(count($objects) > 0) {
$G = 0;
$objects = $this->setupWeights($objects, $this->words, $this->properties[$class]["table"], $G);
//$GLOBALS['timer']->sample("objects weighed");
foreach($objects as $n => $object) {
$objects[$n]->relevance = $this->calculateRelevance($object, $G);
}
//$GLOBALS['timer']->sample("objects ranked");
usort($objects, "order_objects_by_relevance_desc");
if($from !== null && $to !== null) {
$results[$class] = array_slice ($objects, $from, $to-$from + 1);
}
else {
$results[$class] = $objects;
}
}
}
//$GLOBALS['timer']->sample("objects ordered");
return $results;
}
function getObjectFtVars($object, $ft_fields) {
$sw = !isset($GLOBALS['stopwords']) ? FtLikeSearch::setStopwords(SEARCH_STOPWORDS_FILE)
: is_array($GLOBALS['stopwords']);
$fulltext = "";
foreach ($ft_fields as $field) {
eval("\$data = \$object->get".ucfirst(camelCase($field))."();");
$words = str_replace(array("-", "\r\n", "\n", "\r"), " ", strtolower(strip_tags($data)));
$words = preg_replace("/[^a-z0-9\s\t]/", "", $words);
if($sw) {
$words = preg_replace($GLOBALS['stopwords'], " ", $words);
}
$fulltext.= $words." ";
}
$unique_words = FtLikeSearch::getUniqueWords($fulltext);
$sum_dtf = FtLikeSearch::getSumDtf($fulltext, $unique_words);
$U = sizeof($unique_words);
$norm_pivot = $U/(1 + 0.0115 * $U);
return array("fulltext" => $fulltext, "sum_dtf" => $sum_dtf, "norm_pivot" => $norm_pivot );
}
function getUniqueWords($fulltext) {
return array_keys(array_flip(str_word_count($fulltext, 1)));
}
function getSumDtf($fulltext, $unique_words) {
$sum_dtf = 0;
for($n = 0; $n < count($unique_words); $n++) {
$dtf = substr_count($fulltext, $unique_words[$n]);
$sum_dtf += log((float)$dtf) + 1;
}
return $sum_dtf;
}
function setupWeights($objects, $words, $table, & $G) {
$lcquery = strtolower($this->query);
for($n = 0; $n < count($objects); $n++) {
$objects[$n]->LNqf = array();
$ft = $objects[$n]->ft;
$sum_dtf = $objects[$n]->sum_dtf;
$norm_pivot = $objects[$n]->norm_pivot;
foreach($words as $i => $word) {
foreach($this->variants[$i] as $type => $variant) {
$dtf = substr_count($ft, $variant);
if($dtf > 0) break;
}
$vf = $this->variantFactors[$type];
$qf = substr_count($lcquery, $word);
$L = (log((float)$dtf) + 1) / $sum_dtf;
$N = $norm_pivot;
$objects[$n]->LNqf[] = $L * $N * $qf * $vf;
}
}
$row = mysql_query("SELECT count(*) AS rows FROM ".$table." LIMIT 1") or die (mysql_error());
$rowCountResult = mysql_fetch_object($row);
$N = $rowCountResult->rows;
$nf = count($objects);
$G = log((float)(($N - $nf) / $nf)); // IDFP;
// $G = log((float)($N / $nf)); // IDF;
return $objects;
}
function calculateRelevance($object, $G) {
$relevance = 0;
foreach($object->LNqf as $LNqf) {
$R = $LNqf * $G;
if(sizeof($object->LNqf) == 1 && $R < 0) $R = -$R;
if(is_finite($R))
$relevance += $R;
}
return $relevance;
}
}
?> |
:strip_icc():strip_exif()/u/74536/demontpx.jpeg?f=community)
/u/107051/jeroenmadtrix60.png?f=community)
:strip_icc():strip_exif()/u/43400/avatar.jpeg?f=community){kind=avatar}
/u/21498/minimoog.png?f=community)
:strip_exif()/u/82924/avatar.gif?f=community){kind=avatar}
/u/661/crop5cca8f904dc18_cropped.png?f=community)
/u/10874/sp.png?f=community)
