add global share string table

Author: cloudwu

3rd/lua/Makefile

@@ -19,7 +19,7 @@ SYSCFLAGS=
 SYSLDFLAGS=
 SYSLIBS=
 
-MYCFLAGS=
+MYCFLAGS=-I../../skynet-src
 MYLDFLAGS=
 MYLIBS=
 MYOBJS=

3rd/lua/lapi.c

@@ -1000,7 +1000,7 @@ static Proto * cloneproto (lua_State *L, const Proto *src) {
     const TValue *s=&src->k[i];
     TValue *o=&f->k[i];
     if (ttisstring(s)) {
-      TString * str = luaS_newlstr(L,svalue(s),vslen(s));
+      TString * str = luaS_clonestring(L,tsvalue(s));
       setsvalue2n(L,o,str);
     } else {
       setobj(L,o,s);
@@ -1288,7 +1288,7 @@ static UpVal **getupvalref (lua_State *L, int fidx, int n, LClosure **pf) {
   StkId fi = index2addr(L, fidx);
   api_check(L, ttisLclosure(fi), "Lua function expected");
   f = clLvalue(fi);
-  api_check(L, (1 <= n && n <= f->p->sizeupvalues), "invalid upvalue index");
+  api_check(L, (1 <= n && n <= f->p->sp->sizeupvalues), "invalid upvalue index");
   if (pf) *pf = f;
   return &f->upvals[n - 1];  /* get its upvalue pointer */
 }

3rd/lua/lgc.c

@@ -188,7 +188,8 @@ void luaC_upvalbarrier_ (lua_State *L, UpVal *uv) {
 
 void luaC_fix (lua_State *L, GCObject *o) {
   global_State *g = G(L);
-  lua_assert(g->allgc == o);  /* object must be 1st in 'allgc' list! */
+  if (g->allgc != o)
+	  return;  /* if object is not 1st in 'allgc' list, it is in global short string table */
   white2gray(o);  /* they will be gray forever */
   g->allgc = o->next;  /* remove object from 'allgc' list */
   o->next = g->fixedgc;  /* link it to 'fixedgc' list */

3rd/lua/lstring.c

@@ -148,10 +148,9 @@ void luaS_remove (lua_State *L, TString *ts) {
 /*
 ** checks whether short string exists and reuses it or creates a new one
 */
-static TString *internshrstr (lua_State *L, const char *str, size_t l) {
+static TString *queryshrstr (lua_State *L, const char *str, size_t l, unsigned int h) {
   TString *ts;
   global_State *g = G(L);
-  unsigned int h = luaS_hash(str, l, g->seed);
   TString **list = &g->strt.hash[lmod(h, g->strt.size)];
   for (ts = *list; ts != NULL; ts = ts->u.hnext) {
     if (l == ts->shrlen &&
@@ -162,6 +161,13 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
       return ts;
     }
   }
+  return NULL;
+}
+
+static TString *addshrstr (lua_State *L, const char *str, size_t l, unsigned int h) {
+  TString *ts;
+  global_State *g = G(L);
+  TString **list = &g->strt.hash[lmod(h, g->strt.size)];
   if (g->strt.nuse >= g->strt.size && g->strt.size <= MAX_INT/2) {
     luaS_resize(L, g->strt.size * 2);
     list = &g->strt.hash[lmod(h, g->strt.size)];  /* recompute with new size */
@@ -174,6 +180,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
   return ts;
 }
 
+static TString *internshrstr (lua_State *L, const char *str, size_t l);
 
 /*
 ** new string (with explicit length)
@@ -224,3 +231,216 @@ Udata *luaS_newudata (lua_State *L, size_t s) {
   return u;
 }
 
+/*
+ * global shared table
+ */
+
+#include "rwlock.h"
+#include "atomic.h"
+#include <stdlib.h>
+
+#define SHRSTR_SLOT 0x10000
+#define HASH_NODE(h) ((h) % SHRSTR_SLOT)
+
+struct shrmap_slot {
+	struct rwlock lock;
+	TString *str;
+};
+
+struct shrmap {
+	struct shrmap_slot h[SHRSTR_SLOT];
+	int n;
+};
+
+static struct shrmap *SSM = NULL;
+
+LUA_API void
+luaS_initshr() {
+	struct shrmap * s = malloc(sizeof(*s));
+	memset(s, 0, sizeof(*s));
+	int i;
+	for (i=0;i<SHRSTR_SLOT;i++) {
+		rwlock_init(&s->h[i].lock);
+	}
+	SSM = s;
+}
+
+LUA_API void
+luaS_exitshr() {
+	int i;
+	for (i=0;i<SHRSTR_SLOT;i++) {
+		TString *str = SSM->h[i].str;
+		while (str) {
+			TString * next = str->u.hnext;
+			free(str);
+			str = next;
+		}
+	}
+	free(SSM);
+}
+
+static TString *
+query_string(unsigned int h, const char *str, lu_byte l) {
+	struct shrmap_slot *s = &SSM->h[HASH_NODE(h)];
+	rwlock_rlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		if (ts->hash == h && 
+			ts->shrlen == l &&
+			memcmp(str, ts+1, l) == 0) {
+			break;
+		}
+		ts = ts->u.hnext;
+	}
+	rwlock_runlock(&s->lock);
+	return ts;
+}
+
+static TString *
+query_ptr(TString *t) {
+	unsigned int h = t->hash;
+	struct shrmap_slot *s = &SSM->h[HASH_NODE(h)];
+	rwlock_rlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		if (ts == t)
+			break;
+		ts = ts->u.hnext;
+	}
+	rwlock_runlock(&s->lock);
+	return ts;
+}
+
+static TString *
+new_string(unsigned int h, const char *str, lu_byte l) {
+	size_t sz = sizelstring(l);
+	TString *ts = malloc(sz);
+	memset(ts, 0, sz);
+	ts->tt = LUA_TSHRSTR;
+	ts->hash = h;
+	ts->shrlen = l;
+	memcpy(ts+1, str, l);
+	return ts;
+}
+
+static TString *
+add_string(unsigned int h, const char *str, lu_byte l) {
+	TString * tmp = new_string(h, str, l);
+	struct shrmap_slot *s = &SSM->h[HASH_NODE(h)];
+	rwlock_wlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		if (ts->hash == h && 
+			ts->shrlen == l &&
+			memcmp(str, ts+1, l) == 0) {
+				break;
+		}
+		ts = ts->u.hnext;
+	}
+	if (ts == NULL) {
+		ts = tmp;
+		ts->u.hnext = s->str;
+		s->str = ts;
+		tmp = NULL;
+	}
+	rwlock_wunlock(&s->lock);
+	if (tmp) {
+		// string is create by other thread, so free tmp
+		free(tmp);
+	}
+	return ts;
+}
+
+static TString *
+internshrstr (lua_State *L, const char *str, size_t l) {
+  TString *ts;
+  global_State *g = G(L);
+  unsigned int h = luaS_hash(str, l, g->seed);
+  unsigned int h0;
+  // lookup global state of this L first
+  ts = queryshrstr (L, str, l, h);
+  if (ts)
+    return ts;
+  // lookup SSM again
+  h0 = luaS_hash(str, l, 0);
+  ts = query_string(h0, str, l);
+  if (ts)
+    return ts;
+  // If SSM->n greate than 0, add it to SSM
+  if (SSM->n > 0) {
+    ATOM_DEC(&SSM->n);
+    return add_string(h0, str, l);
+  }
+  // Else add it to global state (local)
+  return addshrstr (L, str, l, h);
+}
+
+LUA_API void
+luaS_expandshr(int n) {
+  ATOM_ADD(&SSM->n, n);
+}
+
+LUAI_FUNC TString *
+luaS_clonestring(lua_State *L, TString *ts) {
+  unsigned int h;
+  int l;
+  const char * str = getaddrstr(ts);
+  global_State *g = G(L);
+  TString *result;
+  if (ts->tt == LUA_TLNGSTR) 
+    return luaS_newlstr(L, str, ts->u.lnglen);
+  // look up global state of this L first
+  l = ts->shrlen;
+  h = luaS_hash(str, l, g->seed);
+  result = queryshrstr (L, str, l, h);
+  if (result)
+    return result;
+  // look up SSM by ptr
+  result = query_ptr(ts);
+  if (result)
+    return result;
+  // ts is not in SSM, so recalc hash, and add it to SSM
+  h = luaS_hash(str, l, 0);
+  return add_string(h, str, l);
+}
+
+struct slotinfo {
+	int len;
+	int size;
+};
+
+static void
+getslot(struct shrmap_slot *s, struct slotinfo *info) {
+	memset(info, 0, sizeof(*info));
+	rwlock_rlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		++info->len;
+		info->size += ts->shrlen;
+		ts = ts->u.hnext;
+	}
+	rwlock_runlock(&s->lock);
+}
+
+LUA_API int
+luaS_shrinfo(lua_State *L) {
+	struct slotinfo total;
+	struct slotinfo tmp;
+	memset(&total, 0, sizeof(total));
+	int i;
+	int len = 0;
+	for (i=0;i<SHRSTR_SLOT;i++) {
+		struct shrmap_slot *s = &SSM->h[i];
+		getslot(s, &tmp);
+		len += tmp.len;
+		if (tmp.len > total.len) {
+			total.len = tmp.len;
+		}
+		total.size += tmp.size;
+	}
+	lua_pushinteger(L, len);
+	lua_pushinteger(L, total.size);
+	lua_pushinteger(L, total.len);
+	lua_pushinteger(L, SSM->n);
+	return 4;
+}

3rd/lua/lstring.h

@@ -43,5 +43,10 @@ LUAI_FUNC Udata *luaS_newudata (lua_State *L, size_t s);
 LUAI_FUNC TString *luaS_newlstr (lua_State *L, const char *str, size_t l);
 LUAI_FUNC TString *luaS_new (lua_State *L, const char *str);
 
+LUA_API void luaS_initshr();
+LUA_API void luaS_exitshr();
+LUA_API void luaS_expandshr(int n);
+LUAI_FUNC TString *luaS_clonestring(lua_State *L, TString *);
+LUA_API int luaS_shrinfo(lua_State *L);
 
 #endif

3rd/lua/luaconf.h

@@ -11,6 +11,7 @@
 #include <limits.h>
 #include <stddef.h>
 
+#define LUA_USE_APICHECK
 
 /*
 ** ===================================================================

lualib-src/lua-memory.c

@@ -2,6 +2,7 @@
 #include <lauxlib.h>
 
 #include "malloc_hook.h"
+#include "luashrtbl.h"
 
 static int
 ltotal(lua_State *L) {
@@ -33,6 +34,13 @@ ldump(lua_State *L) {
 	return 0;
 }
 
+static int
+lexpandshrtbl(lua_State *L) {
+	int n = luaL_checkinteger(L, 1);
+	luaS_expandshr(n);
+	return 0;
+}
+
 int
 luaopen_memory(lua_State *L) {
 	luaL_checkversion(L);
@@ -43,6 +51,8 @@ luaopen_memory(lua_State *L) {
 		{ "dumpinfo", ldumpinfo },
 		{ "dump", ldump },
 		{ "info", dump_mem_lua },
+		{ "ssinfo", luaS_shrinfo },
+		{ "ssexpand", lexpandshrtbl },
 		{ NULL, NULL },
 	};
 

service/bootstrap.lua

@@ -1,8 +1,12 @@
 local skynet = require "skynet"
 local harbor = require "skynet.harbor"
 require "skynet.manager"	-- import skynet.launch, ...
+local memory = require "memory"
 
 skynet.start(function()
+	local sharestring = tonumber(skynet.getenv "sharestring")
+	memory.ssexpand(sharestring or 4096)
+
 	local standalone = skynet.getenv "standalone"
 
 	local launcher = assert(skynet.launch("snlua","launcher"))

service/debug_console.lua

@@ -3,6 +3,7 @@ local codecache = require "skynet.codecache"
 local core = require "skynet.core"
 local socket = require "socket"
 local snax = require "snax"
+local memory = require "memory"
 
 local port = tonumber(...)
 local COMMAND = {}
@@ -130,6 +131,8 @@ function COMMAND.help()
 		log = "launch a new lua service with log",
 		debug = "debug address : debug a lua service",
 		signal = "signal address sig",
+		cmem = "Show C memory info",
+		shrtbl = "Show shared short string table info",
 	}
 end
 
@@ -258,3 +261,19 @@ function COMMAND.signal(address, sig)
 		core.command("SIGNAL", address)
 	end
 end
+
+function COMMAND.cmem()
+	local info = memory.info()
+	local tmp = {}
+	for k,v in pairs(info) do
+		tmp[skynet.address(k)] = v
+	end
+	return tmp
+end
+
+function COMMAND.shrtbl()
+	local n, total, longest, space = memory.ssinfo()
+	return { n = n, total = total, longest = longest, space = space }
+end
+
+