Index: server/modules/tags/rxmlparse.pike
===================================================================
RCS file: /cvs/Roxen/5.0/server/modules/tags/rxmlparse.pike,v
retrieving revision 1.81
diff -u -r1.81 rxmlparse.pike
--- server/modules/tags/rxmlparse.pike 5 Nov 2008 20:13:46 -0000 1.81
+++ server/modules/tags/rxmlparse.pike 18 Jan 2009 13:27:52 -0000
@@ -60,6 +60,10 @@
the RXML pages in RAM when this is enabled, which speeds up the
evaluation of them.");
+ defvar("disk_cache_pages", 1, "Cache parsed RXML pages on disk",
+ TYPE_FLAG, #"\
+The RXML parser will cache the RXML parse trees on disk.");
+
defvar("logerrorsp", 0, "RXML Errors:Log RXML parse errors", TYPE_FLAG,
"If enabled, all RXML parse errors will be logged in the debug log.");
@@ -85,6 +89,14 @@
require_exec=[int]query("require_exec");
parse_exec=[int]query("parse_exec");
ram_cache_name = query ("ram_cache_pages") && "p-code:" + c->name;
+ if (query ("disk_cache_pages")) {
+ if (!disk_cache)
+ disk_cache = Cache.cache (
+ Cache.Storage.Gdbm ((getenv ("VARDIR") || "../var") + "/" +
+ c->name + "_rxmlcode"),
+ Cache.Policy.Sized (50 * 1024 * 1024));
+ }
+ else disk_cache = 0;
c->rxml_tag_set->handle_run_error = rxml_run_error;
c->rxml_tag_set->handle_parse_error = rxml_parse_error;
}
@@ -99,19 +111,36 @@
int require_exec, parse_exec;
int bytes; // Holds the number of bytes parsed
-int ram_cache_pages;
string ram_cache_name;
function(string,int|void,string|void:string) file2type;
-mapping handle_file_extension(Stdio.File file, string e, RequestID id)
-{
- Stdio.Stat stat = id->misc->stat || file->stat();
+Cache.cache disk_cache;
+mapping(string:int) is_storing = ([]);
- if(require_exec && !(stat[0] & 07111)) return 0;
- if(!parse_exec && (stat[0] & 07111)) return 0;
-
- bytes += stat[1];
+void encode_and_store (string key, int mtime, RXML.PCode p_code)
+{
+ string p_code_str;
+ if (mixed err = catch {
+ p_code_str = RXML.p_code_to_string (p_code, my_configuration());
+ werror ("p-code size for %O: %d\n", key, sizeof (p_code_str));
+ }) {
+#ifdef DEBUG
+ report_notice ("Failed to encode disk cache entry for %O:\n%s",
+ key, describe_backtrace (err));
+#else
+ report_notice ("Failed to encode disk cache entry for %O: %s",
+ key, describe_error (err));
+#endif
+ // Store a placeholder to avoid trying to compile this again.
+ disk_cache->store (key, ({mtime, 0}));
+ }
+ else
+ disk_cache->store (key, ({mtime, p_code_str}));
+ m_delete (is_storing, key);
+}
+string get_rxml_source (Stdio.File file, RequestID id)
+{
string data = file->read();
switch( id->misc->input_charset )
{
@@ -130,53 +159,126 @@
->drain());
break;
}
+ werror ("source size for %O: %d\n", id->not_query, sizeof (data));
+ return data;
+}
+
+mapping handle_file_extension(Stdio.File file, string e, RequestID id)
+{
+ Stdio.Stat stat = id->misc->stat || file->stat();
+
+ if(require_exec && !(stat[0] & 07111)) return 0;
+ if(!parse_exec && (stat[0] & 07111)) return 0;
+
+ bytes += stat[1];
RXML.Context context;
string rxml;
#ifdef MAY_OVERRIDE_RXML_PARSING
if(id->prestate->norxml)
- rxml = data;
+ rxml = get_rxml_source (file, id);
else
#endif
{
+ RXML.PCode p_code;
+
eval_rxml:
- if (ram_cache_name) {
- array cache_ent;
- if ((cache_ent = cache_lookup (ram_cache_name, id->not_query)) &&
- cache_ent[0] == stat[ST_MTIME]) {
- TRACE_ENTER (sprintf ("Evaluating RXML page %O from RAM cache",
- id->not_query), this_object());
- if (cache_ent[1]->is_stale()) {
- cache_remove (ram_cache_name, id->not_query);
- TRACE_LEAVE ("RAM cache entry was stale");
+ {
+ int try_compile = 0;
+
+ if (ram_cache_name) {
+ array cache_ent;
+ if ((cache_ent = cache_lookup (ram_cache_name, id->not_query)) &&
+ cache_ent[0] == stat[ST_MTIME]) {
+ SIMPLE_TRACE_ENTER (this, "Evaluating RXML page %O from RAM cache",
+ id->not_query);
+ if (cache_ent[1]->is_stale()) {
+ cache_remove (ram_cache_name, id->not_query);
+ SIMPLE_TRACE_LEAVE ("RAM cache entry was stale");
+ }
+ else {
+ p_code = cache_ent[1];
+ context = p_code->new_context (id);
+ rxml = p_code->eval (context);
+ id->cache_status["pcoderam"] = 1;
+ break eval_rxml;
+ }
}
- else {
- context = cache_ent[1]->new_context (id);
- rxml = cache_ent[1]->eval (context);
- id->cache_status["pcoderam"] = 1;
- break eval_rxml;
+ try_compile = 1;
+ }
+
+ if (disk_cache) {
+ array cache_ent;
+ if ((cache_ent = disk_cache->lookup (id->not_query)) &&
+ cache_ent[0] == stat[ST_MTIME]) {
+ if (cache_ent[1]) {
+ SIMPLE_TRACE_ENTER (this, "Decoding RXML page %O from disk cache",
+ id->not_query);
+ if (mixed err = catch {
+ p_code = RXML.string_to_p_code (cache_ent[1], my_configuration());
+ }) {
+#ifdef DEBUG
+ report_warning ("Failed to decode disk cache entry for %O:\n%s",
+ id->not_query, describe_backtrace (err));
+#elif defined (MODULE_DEBUG)
+ report_warning ("Failed to decode disk cache entry for %O:\n%s",
+ id->not_query, describe_error (err));
+#endif
+ SIMPLE_TRACE_LEAVE ("");
+ }
+ else {
+ if (ram_cache_name)
+ cache_set (ram_cache_name, id->not_query, ({stat[ST_MTIME], p_code}));
+ SIMPLE_TRACE_LEAVE ("");
+ SIMPLE_TRACE_ENTER (this, "Evaluating p-code for RXML page %O",
+ id->not_query);
+ context = p_code->new_context (id);
+ rxml = p_code->eval (context); // p_code can't be stale here.
+ id->cache_status["pcodedisk"] = 1;
+ break eval_rxml;
+ }
+ try_compile = 1;
+ }
+ else {
+ // Don't set try_compile; we don't want to try to compile
+ // this again since it failed to encode the last time.
+ try_compile = 1;
+ }
}
+ else try_compile = 1;
+ }
+
+ if (try_compile) {
+ SIMPLE_TRACE_ENTER (this, "Evaluating and compiling RXML page %O",
+ id->not_query);
+ RXML.Parser parser = Roxen.get_rxml_parser (id, 0, 1);
+ context = parser->context;
+ parser->write_end (get_rxml_source (file, id));
+ rxml = parser->eval();
+ p_code = parser->p_code;
+ p_code->finish();
+ if (ram_cache_name)
+ cache_set (ram_cache_name, id->not_query, ({stat[ST_MTIME], p_code}));
+ }
+ else {
+ SIMPLE_TRACE_ENTER (this, "Evaluating RXML page %O", id->not_query);
+ RXML.Parser parser = Roxen.get_rxml_parser (id);
+ context = parser->context;
+ parser->write_end (get_rxml_source (file, id));
+ rxml = parser->eval();
}
- TRACE_ENTER (sprintf ("Evaluating and compiling RXML page %O",
- id->not_query), this_object());
- RXML.Parser parser = Roxen.get_rxml_parser (id, 0, 1);
- context = parser->context;
- parser->write_end (data);
- rxml = parser->eval();
- RXML.PCode p_code = parser->p_code;
- p_code->finish();
- cache_set (ram_cache_name, id->not_query, ({stat[ST_MTIME], p_code}));
}
- else {
- TRACE_ENTER (sprintf ("Evaluating RXML page %O",
- id->not_query), this_object());
- RXML.Parser parser = Roxen.get_rxml_parser (id);
- context = parser->context;
- parser->write_end (data);
- rxml = parser->eval();
+
+ SIMPLE_TRACE_LEAVE ("");
+
+ if (disk_cache && p_code && !is_storing[id->not_query] &&
p_code->is_updated()) {
+ is_storing[id->not_query] = 1;
+ // Encode and store the p-code in background to avoid
+ // unnecessary delay in delivering the response.
+ roxen.background_run (
+ 0, encode_and_store, id->not_query, stat[ST_MTIME], p_code);
}
}
- TRACE_LEAVE ("");
return (["data":rxml,
"type": file2type((id->realfile
|