From 39551c85d6c07bc20b8756ff41de3f85a3cd57b8 Mon Sep 17 00:00:00 2001 From: Heikki Levanto Date: Wed, 18 Dec 2013 16:18:22 +0100 Subject: [PATCH] Fitting: better initial guess The two test cases now converge to within two decimals in 19 and 46 iterations, which must be good enough. The results look reasonable. --- src/relevance.c | 36 +++++++++++++++++++++++------------- test/test_rank_12.res | 38 +++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/relevance.c b/src/relevance.c index 4cbf7f2..02dc971 100644 --- a/src/relevance.c +++ b/src/relevance.c @@ -165,8 +165,8 @@ static void setup_norm_record( struct relevance *rel, struct record_cluster *cl } else { if ( rp->score > norm->max ) norm->max = rp->score; - if ( rp->score < norm->min && abs(rp->score) < 1e-6 ) - norm->min = rp->score; // skip zeroes + if ( rp->score < norm->min ) + norm->min = rp->score; } } } @@ -190,14 +190,15 @@ static double squaresum( struct norm_record *rp, double a, double b) static void normalize_scores(struct relevance *rel) { const int maxiterations = 1000; - const double enough = 1000.0; // sets the number of decimals we are happy with + const double enough = 100.0; // sets the number of decimals we are happy with const double stepchange = 0.5; // reduction of the step size when finding middle // 0.5 sems to be magical, much better than 0.4 or 0.6 struct norm_client *norm; for ( norm = rel->norm; norm; norm = norm->next ) { - yaz_log(YLOG_LOG,"Normalizing client %d: scorefield=%d count=%d range=%f %f", - norm->num, norm->scorefield, norm->count, norm->min, norm->max); + yaz_log(YLOG_LOG,"Normalizing client %d: scorefield=%d count=%d range=%f %f = %f", + norm->num, norm->scorefield, norm->count, norm->min, + norm->max, norm->max-norm->min); norm->a = 1.0; // default normalizing factors, no change norm->b = 0.0; if ( norm->scorefield != scorefield_none && @@ -210,13 +211,26 @@ static void normalize_scores(struct relevance *rel) double chi; char *branch = "?"; // initial guesses for the parameters + // Rmax = a * rmax + b # want to be 1.0 + // Rmin = a * rmin + b # want to be 0.0 + // Rmax - Rmin = a ( rmax - rmin ) # subtracting equations + // 1.0 - 0.0 = a ( rmax - rmin ) + // a = 1 / range + // Rmin = a * rmin + b + // b = Rmin - a * rmin + // = 0.0 - 1/range * rmin + // = - rmin / range + if ( range < 1e-6 ) // practically zero range = norm->max; a = 1.0 / range; - b = abs(norm->min); + b = -1.0 * norm->min / range; + // b = fabs(norm->min) / range; as = a / 10; - bs = b / 10; + bs = fabs(b) / 10; chi = squaresum( norm->records, a,b); + yaz_log(YLOG_LOG,"Initial done: it=%d: a=%f / %f b=%f / %f chi = %f", + 0, a, as, b, bs, chi ); while (it++ < maxiterations) // safeguard against things not converging { double aplus = squaresum(norm->records, a+as, b); @@ -269,7 +283,7 @@ static void normalize_scores(struct relevance *rel) branch = "step b"; } } - yaz_log(YLOG_LOG,"Fitting %s it=%d: a=%f %f b=%f %f chi=%f ap=%f am=%f, bp=%f bm=%f p=%f", + yaz_log(YLOG_LOG,"Fitting %s it=%d: a=%g %g b=%g %g chi=%g ap=%g am=%g, bp=%g bm=%g p=%g", branch, it, a, as, b, bs, chi, aplus, aminus, bplus, bminus, prevchi ); norm->a = a; @@ -280,12 +294,8 @@ static void normalize_scores(struct relevance *rel) } } - yaz_log(YLOG_LOG,"Fitting done: it=%d: a=%f / %f b=%f / %f chi = %f", + yaz_log(YLOG_LOG,"Fitting done: it=%d: a=%g / %g b=%g / %g chi = %g", it-1, a, as, b, bs, chi ); - yaz_log(YLOG_LOG," a: %f < %f %d", - fabs(as)*enough, fabs(a), (fabs(as) * enough < fabs(a)) ); - yaz_log(YLOG_LOG," b: %f < %f %d", - fabs(bs)*enough, fabs(b), (fabs(bs) * enough < fabs(b)) ); } if ( norm->scorefield != scorefield_none ) diff --git a/test/test_rank_12.res b/test/test_rank_12.res index f03b6fc..43c4128 100644 --- a/test/test_rank_12.res +++ b/test/test_rank_12.res @@ -21,7 +21,7 @@ 2.304635 1 - 6645 + 8294 content: title water management problems and challenges in india author dinesh kumar m medium book @@ -40,7 +40,7 @@ 2.231453 1 - 5215 + 6424 content: title the magic of water author hochschwender ted medium book @@ -57,7 +57,7 @@ 2.186368 1 - 4334 + 5273 content: title water author de villiers marq medium book @@ -74,7 +74,7 @@ 2.186368 1 - 4334 + 5273 content: title water use for public water supply in michigan medium book @@ -91,7 +91,7 @@ 2.114981 1 - 2939 + 3449 content: title report to the iucn on water demand management country study medium book @@ -111,7 +111,7 @@ 2.061328 1 - 1890 + 2078 content: title evaluation and control of water pollution in bhavani basin medium book @@ -130,7 +130,7 @@ 2.061328 1 - 1890 + 2078 content: title unsia water cluster medium book @@ -147,7 +147,7 @@ 2.061328 1 - 1890 + 2078 content: title water and water supplies author thresh john clough medium book @@ -166,7 +166,7 @@ 2.061328 1 - 1890 + 2078 content: title water author majeed abdul medium book @@ -185,7 +185,7 @@ 2.061328 1 - 1890 + 2078 content: title water law author fisher d e medium book @@ -202,7 +202,7 @@ 2.061328 1 - 1890 + 2078 content: title water technology management medium book @@ -219,7 +219,7 @@ 2.037029 1 - 1415 + 1457 content: title wonderful water author glover david medium book @@ -239,7 +239,7 @@ 2.016555 1 - 1015 + 934 content: title a primer on fresh water medium book @@ -266,7 +266,7 @@ 2.016555 2 - 1015 + 934 content: title water quality assessment of the state water project medium book @@ -283,7 +283,7 @@ 1.928196 1 - -710 + -1322 content: title district water supply plan medium book @@ -302,7 +302,7 @@ 1.928196 1 - -710 + -1322 content: title proposition medium book @@ -320,7 +320,7 @@ 1.928196 1 - -710 + -1322 content: title wastewater and drinking water user charge survey medium book @@ -340,7 +340,7 @@ 1.928196 1 - -710 + -1322 content: title water in press medium book @@ -359,7 +359,7 @@ 1.928196 1 - -710 + -1322 content: title who governs water author frey hans medium book \ No newline at end of file -- 1.7.10.4