allow running server from LFS (#125)

README.md

@@ -56,7 +56,7 @@ Let the abuse begin.
 3. Visit your server from a web browser.
 
    __Example:__ Say the IP for your ESP8266 is 2.2.2.2 and the server is
-   running in the default port 80. Go to (http://2.2.2.2/index.html)[http://2.2.2.2/index.html] using your web browser.
+   running in the default port 80. Go to <http://2.2.2.2/index.html> using your web browser.
    The ESP8266 will serve you with the contents of the file "http/index.html" (if it exists). If you visit the root (/)
    then index.html is served. By the way, unlike most HTTP servers, nodemcu_httpserver treats the URLs in a
    case-sensitive manner.
@@ -88,6 +88,20 @@ Let the abuse begin.
    For example, if the client requests _http://2.2.2.2/foo.lua?color=red_ then the server will execute the function
    in your Lua script _foo.lua_ and pass in _connection_ and _args_, where _args.color = "red"_.
 
+## [LFS](https://nodemcu.readthedocs.io/en/master/lfs/) support.
+
+   NodeMCU allows to run Lua scripts from LFS in order to save RAM resources.
+   *nodemcu-httpserver* makes it easy to move your code to LFS.
+   In order to run *nodemcu-httpserver* from LFS:
+
+   1. move your code to `srv` folder (if you want it to be included in LFS image)
+
+   1. Compile contents of `srv` into LFS image. There's a [cloud service](https://blog.ellisons.org.uk/article/nodemcu/a-lua-cross-compile-web-service/) and [docker image](https://github.com/marcelstoer/docker-nodemcu-build) that will help you with that.
+
+   1. Upload image file under `lfs.img` name.
+
+   1. Reboot you NodeMCU. Init script will pick up image and apply it for you.
+
 ### Example: Garage door opener
 
 #### Purpose

httpserver-compile.lua

@@ -4,30 +4,37 @@
 
 local compileAndRemoveIfNeeded = function(f)
    if file.exists(f) then
-      print('Compiling:', f)
+      local newf = f:gsub("%w+/", "")
-      node.compile(f)
+      file.rename(f, newf)
-      file.remove(f)
+      print('Compiling:', newf)
+      node.compile(newf)
+      file.remove(newf)
       collectgarbage()
    end
 end
 
 local serverFiles = {
-   'httpserver.lua',
+   'srv/httpserver.lua',
-   'httpserver-b64decode.lua',
+   'srv/httpserver-b64decode.lua',
-   'httpserver-basicauth.lua',
+   'srv/httpserver-basicauth.lua',
-   'httpserver-compile.lua',
+   'srv/httpserver-buffer.lua',
-   'httpserver-conf.lua',
+   'srv/httpserver-connection.lua',
-   'httpserver-connection.lua',
+   'srv/httpserver-error.lua',
-   'httpserver-error.lua',
+   'srv/httpserver-header.lua',
-   'httpserver-header.lua',
+   'srv/httpserver-init.lua',
-   'httpserver-init.lua',
+   'srv/httpserver-request.lua',
-   'httpserver-request.lua',
+   'srv/httpserver-static.lua',
-   'httpserver-static.lua',
+   'srv/httpserver-wifi.lua',
-   'httpserver-wifi.lua',
+}
+
+local lfsFiles = {
+   'srv/_init.lua',
+   'srv/dummy_strings.lua',
 }
 for i, f in ipairs(serverFiles) do compileAndRemoveIfNeeded(f) end
+for i, f in ipairs(lfsFiles) do file.remove(f) end
 
 compileAndRemoveIfNeeded = nil
 serverFiles = nil
+lfsFiles = nil
 collectgarbage()
-

init.lua

@@ -1,9 +1,25 @@
 -- Compile freshly uploaded nodemcu-httpserver lua files.
-if file.exists("httpserver-compile.lc") then
+if node.getpartitiontable().lfs_size > 0 then
-   dofile("httpserver-compile.lc")
+   if file.exists("lfs.img") then
+      if file.exists("lfs_lock") then
+	 file.remove("lfs_lock")
+	 file.remove("lfs.img")
       else
-   dofile("httpserver-compile.lua")
+	 local f = file.open("lfs_lock", "w")
+	 f:flush()
+	 f:close()
+	 file.remove("httpserver-compile.lua")
+	 node.flashreload("lfs.img")
       end
+   end
+   pcall(node.flashindex("_init"))
+end
+
+if file.exists("httpserver-compile.lua") then
+   dofile("httpserver-compile.lua")
+   file.remove("httpserver-compile.lua")
+end
+
 
 -- Set up NodeMCU's WiFi
 dofile("httpserver-wifi.lc")

srv/_init.lua

@@ -0,0 +1,99 @@
+--
+-- File: _init.lua
+--[[
+
+  This is a template for the LFS equivalent of the SPIFFS init.lua.
+
+  It is a good idea to such an _init.lua module to your LFS and do most of the LFS
+  module related initialisaion in this. This example uses standard Lua features to
+  simplify the LFS API.
+
+  The first section adds a 'LFS' table to _G and uses the __index metamethod to
+  resolve functions in the LFS, so you can execute the main function of module
+  'fred' by executing LFS.fred(params), etc. It also implements some standard
+  readonly properties:
+
+  LFS._time    The Unix Timestamp when the luac.cross was executed.  This can be
+               used as a version identifier.
+
+  LFS._config  This returns a table of useful configuration parameters, hence
+                 print (("0x%6x"):format(LFS._config.lfs_base))
+               gives you the parameter to use in the luac.cross -a option.
+
+  LFS._list    This returns a table of the LFS modules, hence
+                 print(table.concat(LFS._list,'\n'))
+               gives you a single column listing of all modules in the LFS.
+
+---------------------------------------------------------------------------------]]
+
+local index = node.flashindex
+
+local lfs_t = {
+  __index = function(_, name)
+      local fn_ut, ba, ma, size, modules = index(name)
+      if not ba then
+        return fn_ut
+      elseif name == '_time' then
+        return fn_ut
+      elseif name == '_config' then
+        local fs_ma, fs_size = file.fscfg()
+        return {lfs_base = ba, lfs_mapped = ma, lfs_size = size,
+                fs_mapped = fs_ma, fs_size = fs_size}
+      elseif name == '_list' then
+        return modules
+      else
+        return nil
+      end
+    end,
+
+  __newindex = function(_, name, value)
+      error("LFS is readonly. Invalid write to LFS." .. name, 2)
+    end,
+
+  }
+
+local G=getfenv()
+G.LFS = setmetatable(lfs_t,lfs_t)
+
+--[[-------------------------------------------------------------------------------
+  The second section adds the LFS to the require searchlist, so that you can
+  require a Lua module 'jean' in the LFS by simply doing require "jean". However
+  note that this is at the search entry following the FS searcher, so if you also
+  have jean.lc or jean.lua in SPIFFS, then this SPIFFS version will get loaded into
+  RAM instead of using. (Useful, for development).
+
+  See docs/en/lfs.md and the 'loaders' array in app/lua/loadlib.c for more details.
+
+---------------------------------------------------------------------------------]]
+
+package.loaders[3] = function(module) -- loader_flash
+  local fn, ba = index(module)
+  return ba and "Module not in LFS" or fn
+end
+
+--[[-------------------------------------------------------------------------------
+  You can add any other initialisation here, for example a couple of the globals
+  are never used, so setting them to nil saves a couple of global entries
+---------------------------------------------------------------------------------]]
+
+G.module       = nil    -- disable Lua 5.0 style modules to save RAM
+package.seeall = nil
+
+--[[-------------------------------------------------------------------------------
+  These replaces the builtins loadfile & dofile with ones which preferentially
+  loads the corresponding module from LFS if present.  Flipping the search order
+  is an exercise left to the reader.-
+---------------------------------------------------------------------------------]]
+
+local lf, df = loadfile, dofile
+G.loadfile = function(n)
+  local mod, ext = n:match("(.*)%.(l[uc]a?)");
+  local fn, ba   = index(mod)
+  if ba or (ext ~= 'lc' and ext ~= 'lua') then return lf(n) else return fn end
+end
+
+G.dofile = function(n)
+  local mod, ext = n:match("(.*)%.(l[uc]a?)");
+  local fn, ba   = index(mod)
+  if ba or (ext ~= 'lc' and ext ~= 'lua') then return df(n) else return fn() end
+end

srv/dummy_strings.lua

@@ -0,0 +1,37 @@
+--
+-- File: LFS_dummy_strings.lua
+--[[
+  luac.cross -f generates a ROM string table which is part of the compiled LFS
+  image. This table includes all strings referenced in the loaded modules.
+
+  If you want to preload other string constants, then one way to achieve this is
+  to include a dummy module in the LFS that references the strings that you want
+  to load. You never need to call this module; it's inclusion in the LFS image is
+  enough to add the strings to the ROM table. Your application can use any strings
+  in the ROM table without incuring any RAM or Lua Garbage Collector (LGC)
+  overhead.
+
+  The local preload example is a useful starting point. However, if you call the
+  following code in your application during testing, then this will provide a
+  listing of the current RAM string table.
+
+do
+  local a=debug.getstrings'RAM'
+  for i =1, #a do a[i] = ('%q'):format(a[i]) end
+  print ('local preload='..table.concat(a,','))
+end
+
+  This will exclude any strings already in the ROM table, so the output is the list
+  of putative strings that you should consider adding to LFS ROM table.
+
+---------------------------------------------------------------------------------]]
+
+local preload = "?.lc;?.lua", "/\n;\n?\n!\n-", "@init.lua", "_G", "_LOADED",
+"_LOADLIB", "__add", "__call", "__concat", "__div", "__eq", "__gc", "__index",
+"__le", "__len", "__lt", "__mod", "__mode", "__mul", "__newindex", "__pow",
+"__sub", "__tostring", "__unm", "collectgarbage", "cpath", "debug", "file",
+"file.obj", "file.vol", "flash", "getstrings", "index", "ipairs", "list", "loaded",
+"loader", "loaders", "loadlib", "module", "net.tcpserver", "net.tcpsocket",
+"net.udpsocket", "newproxy", "package", "pairs", "path", "preload", "reload",
+"require", "seeall", "wdclr", "not enough memory", "sjson.decoder","sjson.encoder",
+"tmr.timer"

srv/httpserver-basicauth.lua

@@ -18,7 +18,7 @@ end
 -- Returns the username if header contains valid credentials,
 -- nil otherwise.
 function basicAuth.authenticate(header)
-   local conf = dofile("httpserver-conf.lc")
+   local conf = dofile("httpserver-conf.lua")
    local credentials_enc = header:match("Authorization: Basic ([A-Za-z0-9+/=]+)")
    if not credentials_enc then
       return nil
@@ -35,7 +35,7 @@ function basicAuth.authenticate(header)
 end
 
 function basicAuth.authErrorHeader()
-   local conf = dofile("httpserver-conf.lc")
+   local conf = dofile("httpserver-conf.lua")
    return "WWW-Authenticate: Basic realm=\"" .. conf.auth.realm .. "\""
 end
 

srv/httpserver-init.lua

@@ -5,7 +5,7 @@
 -- Function for starting the server.
 -- If you compiled the mdns module, then it will also register with mDNS.
 local startServer = function(ip)
-   local conf = dofile('httpserver-conf.lc')
+   local conf = dofile('httpserver-conf.lua')
    if (dofile("httpserver.lc")(conf['general']['port'])) then
       print("nodemcu-httpserver running at:")
       print("   http://" .. ip .. ":" ..  conf['general']['port'])

srv/httpserver-wifi.lua

@@ -2,12 +2,7 @@
 -- Part of nodemcu-httpserver, configures NodeMCU's WiFI in boot.
 -- Author: Marcos Kirsch
 
-local conf = nil
+local conf = dofile("httpserver-conf.lua")
-if file.exists("httpserver-conf.lc") then
-   conf = dofile("httpserver-conf.lc")
-else
-   conf = dofile("httpserver-conf.lua")
-end
 
 wifi.setmode(conf.wifi.mode)
 

srv/httpserver.lua

@@ -103,7 +103,7 @@ return function (port)
 
          local function onReceive(connection, payload)
 --            collectgarbage()
-            local conf = dofile("httpserver-conf.lc")
+            local conf = dofile("httpserver-conf.lua")
             local auth
             local user = "Anonymous"