Implement lazy loading of directories

When the number of files in a project directory is above the max
limit switch back to a mechanism to read directory content only
when the corresponding folder is expanded in the treeview.

When the command core:find-file is invoked the command core:open-file
is executed instead because the complete list of the project's
files is not available.

When a project search is done we search through all the files within
the project dir without indexing them.

Address issues #217 #203 #183.

1 files changed, 113 insertions, 59 deletions

diff --git a/data/core/init.lua b/data/core/init.lua
index fcadea24..e8b26739 100644
--- a/data/core/init.lua
+++ b/data/core/init.lua
@@ -72,6 +72,7 @@ function core.set_project_dir(new_dir, change_project_fn)
     core.project_directories = {}
     core.add_project_directory(new_dir)
     core.project_files = {}
+    core.project_files_limit = false
     core.reschedule_project_scan()
     return true
   end
@@ -99,6 +100,57 @@ local function strip_trailing_slash(filename)
   return filename
 end
 
+local function compare_file(a, b)
+  return a.filename < b.filename
+end
+
+-- "root" will by an absolute path without trailing '/'
+-- "path" will be a path starting with '/' and without trailing '/'
+--    or the empty string.
+--    It will identifies a sub-path within "root.
+-- The current path location will therefore always be: root .. path.
+-- When recursing "root" will always be the same, only "path" will change.
+-- Returns a list of file "items". In eash item the "filename" will be the
+-- complete file path relative to "root" *without* the trailing '/'.
+local function get_directory_files(root, path, t, recursive, begin_hook)
+  if begin_hook then begin_hook() end
+  local size_limit = config.file_size_limit * 10e5
+  local all = system.list_dir(root .. path) or {}
+  local dirs, files = {}, {}
+
+  local entries_count = 0
+  local max_entries = config.max_project_files
+  for _, file in ipairs(all) do
+    if not common.match_pattern(file, config.ignore_files) then
+      local file = path .. PATHSEP .. file
+      local info = system.get_file_info(root .. file)
+      if info and info.size < size_limit then
+        info.filename = strip_leading_path(file)
+        table.insert(info.type == "dir" and dirs or files, info)
+        entries_count = entries_count + 1
+        if recursive and entries_count > max_entries then return nil, entries_count end
+      end
+    end
+  end
+
+  table.sort(dirs, compare_file)
+  for _, f in ipairs(dirs) do
+    table.insert(t, f)
+    if recursive and entries_count <= max_entries then
+      local subdir_t, subdir_count = get_directory_files(root, PATHSEP .. f.filename, t, recursive)
+      entries_count = entries_count + subdir_count
+      f.scanned = true
+    end
+  end
+
+  table.sort(files, compare_file)
+  for _, f in ipairs(files) do
+    table.insert(t, f)
+  end
+
+  return t, entries_count
+end
+
 local function project_scan_thread()
   local function diff_files(a, b)
     if #a ~= #b then return true end
@@ -110,65 +162,16 @@ local function project_scan_thread()
     end
   end
 
-  local function compare_file(a, b)
-    return a.filename < b.filename
-  end
-
-  -- "root" will by an absolute path without trailing '/'
-  -- "path" will be a path starting with '/' and without trailing '/'
-  --    or the empty string.
-  --    It will identifies a sub-path within "root.
-  -- The current path location will therefore always be: root .. path.
-  -- When recursing "root" will always be the same, only "path" will change.
-  -- Returns a list of file "items". In eash item the "filename" will be the
-  -- complete file path relative to "root" *without* the trailing '/'.
-  local function get_files(root, path, t)
-    coroutine.yield()
-    t = t or {}
-    local size_limit = config.file_size_limit * 10e5
-    local all = system.list_dir(root .. path) or {}
-    local dirs, files = {}, {}
-
-    local entries_count = 0
-    local max_entries = config.max_project_files
-    for _, file in ipairs(all) do
-      if not common.match_pattern(file, config.ignore_files) then
-        local file = path .. PATHSEP .. file
-        local info = system.get_file_info(root .. file)
-        if info and info.size < size_limit then
-          info.filename = strip_leading_path(file)
-          table.insert(info.type == "dir" and dirs or files, info)
-          entries_count = entries_count + 1
-          if entries_count > max_entries then break end
-        end
-      end
-    end
-
-    table.sort(dirs, compare_file)
-    for _, f in ipairs(dirs) do
-      table.insert(t, f)
-      if entries_count <= max_entries then
-        local subdir_t, subdir_count = get_files(root, PATHSEP .. f.filename, t)
-        entries_count = entries_count + subdir_count
-      end
-    end
-
-    table.sort(files, compare_file)
-    for _, f in ipairs(files) do
-      table.insert(t, f)
-    end
-
-    return t, entries_count
-  end
-
   while true do
     -- get project files and replace previous table if the new table is
     -- different
-    for i = 1, #core.project_directories do
+    local i = 1
+    while not core.project_files_limit and i <= #core.project_directories do
       local dir = core.project_directories[i]
-      local t, entries_count = get_files(dir.name, "")
+      local t, entries_count = get_directory_files(dir.name, "", {}, true)
       if diff_files(dir.files, t) then
         if entries_count > config.max_project_files then
+          core.project_files_limit = true
           core.status_view:show_message("!", style.accent,
             "Too many files in project directory: stopping reading at "..
             config.max_project_files.." files according to config.max_project_files. "..
@@ -182,6 +185,7 @@ local function project_scan_thread()
       if dir.name == core.project_dir then
         core.project_files = dir.files
       end
+      i = i + 1
     end
 
     -- wait for next scan
@@ -190,6 +194,46 @@ local function project_scan_thread()
 end
 
 
+function core.scan_project_folder(dirname, filename)
+  for _, dir in ipairs(core.project_directories) do
+    if dir.name == dirname then
+      for i, file in ipairs(dir.files) do
+        local file = dir.files[i]
+        if file.filename == filename then
+          if file.scanned then return end
+          local new_files = get_directory_files(dirname, PATHSEP .. filename, {})
+          for j, new_file in ipairs(new_files) do
+            table.insert(dir.files, i + j, new_file)
+          end
+          file.scanned = true
+          return
+        end
+      end
+    end
+  end
+end
+
+
+local function find_project_files_co(root, path)
+  local size_limit = config.file_size_limit * 10e5
+  local all = system.list_dir(root .. path) or {}
+  for _, file in ipairs(all) do
+    if not common.match_pattern(file, config.ignore_files) then
+      local file = path .. PATHSEP .. file
+      local info = system.get_file_info(root .. file)
+      if info and info.size < size_limit then
+        info.filename = strip_leading_path(file)
+        if info.type == "file" then
+          coroutine.yield(root, info)
+        else
+          find_project_files_co(root, PATHSEP .. info.filename)
+        end
+      end
+    end
+  end
+end
+
+
 local function project_files_iter(state)
   local dir = core.project_directories[state.dir_index]
   state.file_index = state.file_index + 1
@@ -204,17 +248,27 @@ end
 
 
 function core.get_project_files()
-  local state = { dir_index = 1, file_index = 0 }
-  return project_files_iter, state
+  if core.project_files_limit then
+    return coroutine.wrap(function()
+      for _, dir in ipairs(core.project_directories) do
+        find_project_files_co(dir.name, "")
+      end
+    end)
+  else
+    local state = { dir_index = 1, file_index = 0 }
+    return project_files_iter, state
+  end
 end
 
 
 function core.project_files_number()
-  local n = 0
-  for i = 1, #core.project_directories do
-    n = n + #core.project_directories[i].files
+  if not core.project_files_limit then
+    local n = 0
+    for i = 1, #core.project_directories do
+      n = n + #core.project_directories[i].files
+    end
+    return n
   end
-  return n
 end