From ca4fb20c00334f0c3f5100473d685817a823ca11 Mon Sep 17 00:00:00 2001 From: Gregor Hartmann Date: Thu, 14 Nov 2019 18:58:56 +0100 Subject: [PATCH] Serve static pages efficiently. Fix #53 (#118) * 1st draft to serve static files faster * Allow serving 5 images in a page the 6th image cannot be served as the esp does not open more than 5 connections at the same time * win the prize * Update comments --- http/cars.lua | 4 ++++ httpserver-buffer.lua | 48 +++++++++++++++++++++++++++++++++++++++++++ httpserver-static.lua | 30 ++++++++------------------- httpserver.lua | 38 +++++++++++++++++++++++++++++++--- 4 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 httpserver-buffer.lua diff --git a/http/cars.lua b/http/cars.lua index 47282cd..7dc61db 100644 --- a/http/cars.lua +++ b/http/cars.lua @@ -31,6 +31,10 @@ return function (connection, req, args) It works with three embedded images of cars, but the server crashes with four. Select the number of cars you want to see below.
Whoever manages to modify nodemcu-httpserver to load all four images without crashing wins a prize!

+

+ OK I guess I win the prize, as now you can load five cars.
+ Cheers HHHartmann +

choose: show one car show two cars diff --git a/httpserver-buffer.lua b/httpserver-buffer.lua new file mode 100644 index 0000000..18b344f --- /dev/null +++ b/httpserver-buffer.lua @@ -0,0 +1,48 @@ +-- httpserver-buffer +-- Part of nodemcu-httpserver, provides a buffer that behaves like a connection object +-- that can handle multiple consecutive send() calls, and buffers small payloads up to 1400 bytes. +-- This is primarily user to collect the send requests done by the head script. +-- The owner is responsible to call getBuffer and send its result +-- Author: Gregor Hartmann + +local Buffer = {} + +-- parameter is the nodemcu-firmware connection +function Buffer:new() + local newInstance = {} + newInstance.size = 0 + newInstance.data = {} + + -- Returns true if there was any data to be sent. + function newInstance:getBuffer() + local buffer = table.concat(self.data, "") + self.data = {} + self.size = 0 + return buffer + end + + function newInstance:getpeer() + return "no peer" + end + + function newInstance:send(payload) + local flushThreshold = 1400 + if (not payload) then print("nop payload") end + local newSize = self.size + payload:len() + if newSize >= flushThreshold then + print("Buffer is full. Cutting off "..newSize-flushThreshold.." chars") + --STEP1: cut out piece from payload to complete threshold bytes in table + local pieceSize = flushThreshold - self.size + if pieceSize then + payload = payload:sub(1, pieceSize) + end + end + table.insert(self.data, payload) + self.size = self.size + #payload + end + + return newInstance + +end + +return Buffer diff --git a/httpserver-static.lua b/httpserver-static.lua index 10f2e70..1362c0b 100644 --- a/httpserver-static.lua +++ b/httpserver-static.lua @@ -1,27 +1,13 @@ -- httpserver-static.lua -- Part of nodemcu-httpserver, handles sending static files to client. --- Author: Marcos Kirsch +-- Author: Gregor Hartmann return function (connection, req, args) - dofile("httpserver-header.lc")(connection, 200, args.ext, args.isGzipped) - -- Send file in little chunks - local bytesRemaining = file.list()[args.file] - -- Chunks larger than 1024 don't work. - -- https://github.com/nodemcu/nodemcu-firmware/issues/1075 - local chunkSize = 1024 - local fileHandle = file.open(args.file) - while bytesRemaining > 0 do - local bytesToRead = 0 - if bytesRemaining > chunkSize then bytesToRead = chunkSize else bytesToRead = bytesRemaining end - local chunk = fileHandle:read(bytesToRead) - connection:send(chunk) - bytesRemaining = bytesRemaining - #chunk - --print(args.file .. ": Sent "..#chunk.. " bytes, " .. bytesRemaining .. " to go.") - chunk = nil - collectgarbage() - end - -- print("Finished sending: ", args.file) - fileHandle:close() - fileHandle = nil - collectgarbage() + + local buffer = dofile("httpserver-buffer.lc"):new() + dofile("httpserver-header.lc")(buffer, req.code or 200, args.ext, args.isGzipped) + -- Send header and return fileInfo + connection:send(buffer:getBuffer()) + + return { file = args.file, sent = 0} end diff --git a/httpserver.lua b/httpserver.lua index 65b4bc8..7615f6b 100644 --- a/httpserver.lua +++ b/httpserver.lua @@ -13,6 +13,7 @@ return function (port) -- We do it in a separate thread because we need to send in little chunks and wait for the onSent event -- before we can send more, or we risk overflowing the mcu's buffer. local connectionThread + local fileInfo local allowStatic = {GET=true, HEAD=true, POST=false, PUT=false, DELETE=false, TRACE=false, OPTIONS=false, CONNECT=false, PATCH=false} @@ -26,6 +27,10 @@ return function (port) end end + local function startServingStatic(connection, req, args) + fileInfo = dofile("httpserver-static.lc")(connection, req, args) + end + local function startServing(fileServeFunction, connection, req, args) connectionThread = coroutine.create(function(fileServeFunction, bufferedConnection, req, args) fileServeFunction(bufferedConnection, req, args) @@ -40,6 +45,7 @@ return function (port) local BufferedConnectionClass = dofile("httpserver-connection.lc") local bufferedConnection = BufferedConnectionClass:new(connection) + BufferedConnectionClass = nil local status, err = coroutine.resume(connectionThread, fileServeFunction, bufferedConnection, req, args) if not status then log(connection, "Error: "..err) @@ -50,7 +56,7 @@ return function (port) end end - local function handleRequest(connection, req) + local function handleRequest(connection, req, handleError) collectgarbage() local method = req.method local uri = req.uri @@ -84,7 +90,8 @@ return function (port) else if allowStatic[method] then uri.args = {file = uri.file, ext = uri.ext, isGzipped = uri.isGzipped} - fileServeFunction = dofile("httpserver-static.lc") + startServingStatic(connection, req, uri.args) + return else uri.args = {code = 405, errorString = "Method not supported", logFunction = log} fileServeFunction = dofile("httpserver-error.lc") @@ -95,7 +102,7 @@ return function (port) end local function onReceive(connection, payload) - collectgarbage() +-- collectgarbage() local conf = dofile("httpserver-conf.lc") local auth local user = "Anonymous" @@ -162,6 +169,27 @@ return function (port) connectionThread = nil collectgarbage() end + elseif fileInfo then + local fileSize = file.list()[fileInfo.file] + -- Chunks larger than 1024 don't work. + -- https://github.com/nodemcu/nodemcu-firmware/issues/1075 + local chunkSize = 512 + local fileHandle = file.open(fileInfo.file) + if fileSize > fileInfo.sent then + fileHandle:seek("set", fileInfo.sent) + local chunk = fileHandle:read(chunkSize) + fileHandle:close() + fileHandle = nil + fileInfo.sent = fileInfo.sent + #chunk + connection:send(chunk) + -- print(fileInfo.file .. ": Sent "..#chunk.. " bytes, " .. fileSize - fileInfo.sent .. " to go.") + chunk = nil + else + log(connection, "closing connetion", "Finished sending: "..fileInfo.file) + connection:close() + fileInfo = nil + end + collectgarbage() end end @@ -172,6 +200,10 @@ return function (port) connectionThread = nil collectgarbage() end + if fileInfo then + fileInfo = nil + collectgarbage() + end end connection:on("receive", onReceive)