Add VFS to enable POSIX file I/O operations (#2333)

* Add VFS to enable POSIX file I/O operations

Enables use of FILE * operations on internal and external storage.  fopen,
fclose, fseek, fprintf, fscanf, etc. supported.

* Add FS/File::stat and support POSIX stat/fstat

cores/rp2040/FS.cpp

@@ -225,6 +225,21 @@ void File::setTimeCallback(time_t (*cb)(void)) {
     _timeCallback = cb;
 }
 
+bool File::stat(FSStat *st) {
+    if (!_p) {
+        return false;
+    }
+    size_t pos = position();
+    seek(0, SeekEnd);
+    st->size = position() - pos;
+    seek(pos, SeekSet);
+    st->blocksize = 0; // Not set here
+    st->ctime = getCreationTime();
+    st->atime = getLastWrite();
+    st->isDir = isDirectory();
+    return true;
+}
+
 File Dir::openFile(const char* mode) {
     if (!_impl) {
         return File();
@@ -455,6 +470,17 @@ bool FS::rename(const String& pathFrom, const String& pathTo) {
     return rename(pathFrom.c_str(), pathTo.c_str());
 }
 
+bool FS::stat(const char *path, FSStat *st) {
+    if (!_impl) {
+        return false;
+    }
+    return _impl->stat(path, st);
+}
+
+bool FS::stat(const String& path, FSStat *st) {
+    return stat(path.c_str(), st);
+}
+
 time_t FS::getCreationTime() {
     if (!_impl) {
         return 0;

cores/rp2040/FS.h

@@ -48,6 +48,14 @@ enum SeekMode {
     SeekEnd = 2
 };
 
+struct FSStat {
+    size_t size;
+    size_t blocksize;
+    time_t ctime;
+    time_t atime;
+    bool isDir;
+};
+
 class File : public Stream {
 public:
     File(FileImplPtr p = FileImplPtr(), FS *baseFS = nullptr) : _p(p), _fakeDir(nullptr), _baseFS(baseFS) {
@@ -119,6 +127,8 @@ public:
     time_t getCreationTime();
     void setTimeCallback(time_t (*cb)(void));
 
+    bool stat(FSStat *st);
+
 protected:
     FileImplPtr _p;
     time_t (*_timeCallback)(void) = nullptr;
@@ -212,6 +222,9 @@ public:
     bool rmdir(const char* path);
     bool rmdir(const String& path);
 
+    bool stat(const char *path, FSStat *st);
+    bool stat(const String& path, FSStat *st);
+
     // Low-level FS routines, not needed by most applications
     bool gc();
     bool check();

cores/rp2040/FSImpl.h

@@ -126,6 +126,7 @@ public:
     virtual bool remove(const char* path) = 0;
     virtual bool mkdir(const char* path) = 0;
     virtual bool rmdir(const char* path) = 0;
+    virtual bool stat(const char *path, FSStat *st) = 0;
     virtual bool gc() {
         return true;    // May not be implemented in all file systems.
     }

docs/fs.rst

@@ -151,6 +151,48 @@ second SPI port, ``SPI1``.  Just use the following call in place of
     SD.begin(cspin, SPI1);
 
 
+Using VFS (Virtual File System) for POSIX support
+-------------------------------------------------
+The ``VFS`` library enables sketches to use standard POSIX file I/O operations using
+standard ``FILE *`` operations.  Include the ``VFS`` library in your application and
+add a call to map the ``VFS.root()`` to your filesystem.  I.e.:
+
+.. code:: cpp
+
+    #include <VFS.h>
+    #include <LittleFS.h>
+
+    void setup() {
+      LittleFS.begin();
+      VFS.root(LittleFS);
+      FILE *fp = fopen("/thisfilelivesonflash.txt", "w");
+      fprintf(fp, "Hello!\n");
+      fclose(fp);
+    }
+
+Multiple filesystems can be ``VFS.map()`` into the VFS namespace under different directory
+names.  For example, the following will make files on ``/sd`` reside on an external\
+SD card and files on ``/lfs`` live in internal flash.
+
+.. code:: cpp
+
+    #include <VFS.h>
+    #include <LittleFS.h>
+    #include <SDFS.h>
+
+    void setup() {
+      LittleFS.begin();
+      SDFS.begin();
+      VFS.map("/lfs", LittleFS);
+      VFS.map("/sd", SDFS);
+      FILE *onSD = fopen("/sd/thislivesonsd.txt", "wb");
+      ....
+    }
+
+See the examples in the ``VFS`` library for more information.
+
+
+
 File system object (LittleFS/SD/SDFS/FatFS)
 -------------------------------------------
 

libraries/FatFS/src/FatFS.h

@@ -117,6 +117,26 @@ public:
         return _mounted ? (FR_OK == f_unlink(path)) : false;
     }
 
+    bool stat(const char *path, FSStat *st) override {
+        if (!_mounted || !path || !path[0]) {
+            return false;
+        }
+        bzero(st, sizeof(*st));
+        FILINFO fno;
+        if (FR_OK != f_stat(path, &fno)) {
+            return false;
+        }
+        st->size = fno.fsize;
+        st->blocksize = 0;
+        st->isDir = (fno.fattrib & AM_DIR) == AM_DIR;
+        if (st->isDir) {
+            st->size = 0;
+        }
+        st->ctime = FatToTimeT(fno.fdate, fno.ftime);
+        st->atime = FatToTimeT(fno.fdate, fno.ftime);
+        return true;
+    }
+
     bool setConfig(const FSConfig &cfg) override {
         if ((cfg._type != FatFSConfig::FSId) || _mounted) {
             DEBUGV("FatFS::setConfig: invalid config or already mounted\n");

libraries/LittleFS/src/LittleFS.h

@@ -237,6 +237,27 @@ public:
         return true;
     }
 
+    bool stat(const char *path, FSStat *st) override {
+        if (!_mounted || !path || !path[0]) {
+            return false;
+        }
+        lfs_info info;
+        if (lfs_stat(&_lfs, path, &info) < 0) {
+            return false;
+        }
+        st->size = info.size;
+        st->blocksize = _blockSize;
+        st->isDir = info.type == LFS_TYPE_DIR;
+        if (st->isDir) {
+            st->size = 0;
+        }
+        if (lfs_getattr(&_lfs, path, 'c', (void *)&st->ctime, sizeof(st->ctime)) != sizeof(st->ctime)) {
+            st->ctime = 0;
+        }
+        st->atime = st->ctime;
+        return true;
+    }
+
     time_t getCreationTime() override {
         time_t t;
         uint32_t t32b;

libraries/SDFS/src/SDFS.h

@@ -148,6 +148,34 @@ public:
 
     bool format() override;
 
+    bool stat(const char *path, FSStat *st) override {
+        if (!_mounted || !path || !path[0]) {
+            return false;
+        }
+        bzero(st, sizeof(*st));
+        File32 f;
+        f = _fs.open(path, O_RDONLY);
+        if (!f) {
+            return false;
+        }
+        st->size = f.fileSize();
+        st->blocksize = clusterSize();
+        st->isDir = f.isDir();
+        if (st->isDir) {
+            st->size = 0;
+        }
+        uint16_t date;
+        uint16_t time;
+        if (f.getCreateDateTime(&date, &time)) {
+            st->ctime = FatToTimeT(date, time);
+        }
+        if (f.getAccessDate(&date)) {
+            st->atime = FatToTimeT(date, 0);
+        }
+        f.close();
+        return true;
+    }
+
     // The following are not common FS interfaces, but are needed only to
     // support the older SD.h exports
     uint8_t type() {

libraries/VFS/examples/VFSMultipleMounts/VFSMultipleMounts.ino

+// Released to the piublic domain by Earle F. Philhower, III in 2024
+
+#include <LittleFS.h>
+#include <VFS.h>
+#include <SPI.h>
+#include <SDFS.h>
+
+// This are GP pins for SPI0 on the Raspberry Pi Pico board, and connect
+// to different *board* level pinouts.  Check the PCB while wiring.
+// Only certain pins can be used by the SPI hardware, so if you change
+// these be sure they are legal or the program will crash.
+// See: https://datasheets.raspberrypi.com/picow/PicoW-A4-Pinout.pdf
+const int _MISO = 4;  // AKA SPI RX
+const int _MOSI = 7;  // AKA SPI TX
+const int _CS = 5;
+const int _SCK = 6;
+// SPI1
+//const int _MISO = 8;  // AKA SPI RX
+//const int _MOSI = 11;  // AKA SPI TX
+//const int _CS = 9;
+//const int _SCK = 10;
+
+void setup() {
+  delay(5000);
+  if (!LittleFS.begin()) {
+    Serial.printf("ERROR: Unable to start LittleFS. Did you select a filesystem size in the menus?\n");
+    return;
+  }
+
+  SDFSConfig cfg;
+  bool sd = false;
+  if (_MISO == 0 || _MISO == 4 || _MISO == 16) {
+    SPI.setRX(_MISO);
+    SPI.setTX(_MOSI);
+    SPI.setSCK(_SCK);
+    SDFS.setConfig(SDFSConfig(_CS, SPI_HALF_SPEED, SPI));
+    sd = SDFS.begin();
+  } else if (_MISO == 8 || _MISO == 12) {
+    SPI1.setRX(_MISO);
+    SPI1.setTX(_MOSI);
+    SPI1.setSCK(_SCK);
+    SDFS.setConfig(SDFSConfig(_CS, SPI_HALF_SPEED, SPI1));
+    sd = SDFS.begin();
+  } else {
+    Serial.println(F("ERROR: Unknown SPI Configuration"));
+  }
+
+  VFS.map("/lfs", LittleFS); // Onboard flash at /lfs
+  if (sd) {
+    VFS.map("/sd", SDFS); // SD card mapped to /sd
+  }
+  VFS.root(LittleFS); // Anything w/o a prefix maps to LittleFS
+
+  Serial.printf("Writing to /lfs/text.txt\n");
+  FILE *f = fopen("/lfs/text.txt", "wb");
+  fwrite("hello littlefs", 14, 1, f);
+  fclose(f);
+
+  if (sd) {
+    Serial.printf("Writing to /sd/test.txt, should not overwrite /lfs/text.txt!\n");
+    f = fopen("/sd/text.txt", "wb");
+    fwrite("hello sdfs", 10, 1, f);
+    fclose(f);
+  }
+
+  f = fopen("/lfs/text.txt", "rb");
+  char buff[33];
+  bzero(buff, 33);
+  fread(buff, 1, 32, f);
+  fclose(f);
+  Serial.printf("READ LFS> '%s'\n", buff);
+
+  if (sd) {
+    f = fopen("/sd/text.txt", "rb");
+    bzero(buff, 33);
+    fread(buff, 1, 32, f);
+    fclose(f);
+    Serial.printf("READ SDFS> '%s'\n", buff);
+  }
+
+  f = fopen("/text.txt", "rb");
+  bzero(buff, 33);
+  fread(buff, 1, 32, f);
+  fclose(f);
+  Serial.printf("READ default FS (LittleFS)> '%s'\n", buff);
+
+  Serial.printf("\nTesting seeking within a file\n");
+  f = fopen("/lfs/text.txt", "rb");
+  for (int i = 0; i < 10; i ++) {
+    fseek(f, i, SEEK_SET);
+    bzero(buff, 33);
+    fread(buff, 1, 32, f);
+    Serial.printf("LFS SEEK %d> '%s'\n", i, buff);
+  }
+  fclose(f);
+
+  Serial.printf("\nTesting fprintf and fgetc from LFS\n");
+  f = fopen("/lfs/printout.txt", "w");
+  for (int i = 0; i < 10; i++) {
+    fprintf(f, "INT: %d\n", i);
+  }
+  fclose(f);
+
+  Serial.printf("----\n");
+  f = fopen("/printout.txt", "r");
+  int x;
+  while ((x = fgetc(f)) >= 0) {
+    Serial.printf("%c", x);
+  }
+  Serial.printf("----\n");
+}
+
+void loop() {
+}

libraries/VFS/keywords.txt

+#######################################
+# Syntax Coloring Map
+#######################################
+
+#######################################
+# Datatypes (KEYWORD1)
+#######################################
+
+VFS	KEYWORD1
+FILE	KEYWORD1
+
+#######################################
+# Methods and Functions (KEYWORD2)
+#######################################
+
+map	KEYWORD1
+root	KEYWORD1
+
+#######################################
+# Constants (LITERAL1)
+#######################################
+

libraries/VFS/library.properties

+name=VFS
+version=1.0
+author=Earle F. Philhower, III <earlephilhower@yahoo.com>
+maintainer=Earle F. Philhower, III <earlephilhower@yahoo.com>
+sentence=Use POSIX fopen/etc. with Arduino filesystems like LittleFS, SD, etc.
+paragraph=Use POSIX fopen/etc. with Arduino filesystems like LittleFS, SD, etc.
+category=Data Storage
+url=https://gifhub.com/earlephilhower/arduino-pico
+architectures=rp2040
+dot_a_linkage=true

libraries/VFS/src/VFS.cpp

+/*
+    VFS wrapper to allow POSIX FILE operations
+
+    Copyright (c) 2024 Earle F. Philhower, III <earlephilhower@yahoo.com>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#include <list>
+#include <map>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <cerrno>
+#include <FSImpl.h>
+#include <FS.h>
+#include "VFS.h"
+
+// Global static to allow non-class POSIX calls to use this info
+typedef struct {
+    const char *path;
+    FS *fs;
+} Entry;
+static FS *root = nullptr;
+static std::list<Entry> mounts;
+static std::map<int, File> files;
+static int fd = 3;
+
+VFSClass::VFSClass() {
+}
+
+
+void VFSClass::root(FS &fs) {
+    ::root = &fs;
+}
+
+void VFSClass::map(const char *path, FS &fs) {
+    Entry e = { strdup(path), &fs };
+    mounts.push_back(e);
+}
+
+static FS *pathToFS(const char **name) {
+    const char *nm = *name;
+    for (auto a : mounts) {
+        if (!strncmp(a.path, nm, strlen(a.path))) {
+            *name += strlen(a.path);
+            return a.fs;
+        }
+    }
+    return ::root;
+}
+
+extern "C" int _open(char *file, int flags, int mode) {
+    (void) mode; // No mode RWX here
+
+    const char *nm = file;
+    auto fs = pathToFS(&nm);
+    if (!fs) {
+        return -1;
+    }
+    const char *md = "r";
+    // Taken from table at https://pubs.opengroup.org/onlinepubs/9699919799/functions/fopen.html
+    flags &= O_RDONLY | O_WRONLY | O_CREAT | O_TRUNC | O_APPEND | O_RDWR;
+    if (flags == O_RDONLY) {
+        md = "r";
+    } else if (flags == (O_WRONLY | O_CREAT | O_TRUNC)) {
+        md = "w";
+    } else if (flags == (O_WRONLY | O_CREAT | O_APPEND)) {
+        md = "a";
+    } else if (flags == O_RDWR) {
+        md = "r+";
+    } else if (flags == (O_RDWR | O_CREAT | O_TRUNC)) {
+        md = "w+";
+    } else if (flags == (O_RDWR | O_CREAT | O_APPEND)) {
+        md = "a+";
+    }
+    File f = fs->open(nm, md);
+    if (!f) {
+        return -1;
+    }
+    files.insert({fd, f});
+    return fd++;
+}
+
+extern "C" ssize_t _write(int fd, const void *buf, size_t count) {
+#if defined DEBUG_RP2040_PORT
+    if (fd < 3) {
+        return DEBUG_RP2040_PORT.write((const char *)buf, count);
+    }
+#endif
+    auto f = files.find(fd);
+    if (f == files.end()) {
+        return 0; // FD not found
+    }
+    return f->second.write((const char *)buf, count);
+}
+
+extern "C" int _close(int fd) {
+    auto f = files.find(fd);
+    if (f == files.end()) {
+        return -1;
+    }
+    f->second.close();
+    files.erase(f);
+    return 0;
+}
+
+extern "C" int _lseek(int fd, int ptr, int dir) {
+    auto f = files.find(fd);
+    if (f == files.end()) {
+        return -1;
+    }
+    SeekMode d = SeekSet;
+    if (dir == SEEK_CUR) {
+        d = SeekCur;
+    } else if (dir == SEEK_END) {
+        d = SeekEnd;
+    }
+    return f->second.seek(ptr, d) ? 0 : 1;
+}
+
+extern "C" int _read(int fd, char *buf, int size) {
+    auto f = files.find(fd);
+    if (f == files.end()) {
+        return -1; // FD not found
+    }
+    return f->second.read((uint8_t *)buf, size);
+}
+
+extern "C" int _unlink(char *name) {
+    auto f = pathToFS((const char **)&name);
+    if (f) {
+        return f->remove(name) ? 0 : -1;
+    }
+    return -1;
+}
+
+extern "C" int _stat(const char *name, struct stat *st) {
+    auto f = pathToFS((const char **)&name);
+    if (f) {
+        fs::FSStat s;
+        if (!f->stat(name, &s)) {
+            return -1;
+        }
+        bzero(st, sizeof(*st));
+        st->st_size = s.size;
+        st->st_blksize = s.blocksize;
+        st->st_ctim.tv_sec = s.ctime;
+        st->st_atim.tv_sec = s.atime;
+        st->st_mode = s.isDir ? S_IFDIR : S_IFREG;
+        return 0;
+    }
+    return -1;
+}
+
+extern "C" int _fstat(int fd, struct stat *st) {
+    auto f = files.find(fd);
+    if (f == files.end()) {
+        return -1; // FD not found
+    }
+    fs::FSStat s;
+    if (!f->second.stat(&s)) {
+        return -1;
+    }
+    bzero(st, sizeof(*st));
+    st->st_size = s.size;
+    st->st_blksize = s.blocksize;
+    st->st_ctim.tv_sec = s.ctime;
+    st->st_ctim.tv_sec = s.ctime;
+    st->st_mode = s.isDir ? S_IFDIR : S_IFREG;
+    return 0;
+}
+
+VFSClass VFS;

libraries/VFS/src/VFS.h

+/*
+    VFS wrapper to allow POSIX FILE operations
+
+    Copyright (c) 2024 Earle F. Philhower, III <earlephilhower@yahoo.com>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#pragma once
+
+#include <FS.h>
+
+class VFSClass {
+public:
+    VFSClass();
+    // No destructor
+    void root(FS &fs);
+    void map(const char *path, FS &fs);
+};
+
+extern VFSClass VFS;

tests/restyle.sh

@@ -16,7 +16,8 @@ for dir in ./cores/rp2040 ./libraries/EEPROM ./libraries/I2S ./libraries/SingleF
            ./libraries/lwIP_w5500 ./libraries/lwIP_w5100 ./libraries/lwIP_enc28j60 \
            ./libraries/SPISlave ./libraries/lwIP_ESPHost ./libraries/FatFS\
            ./libraries/FatFSUSB ./libraries/BluetoothAudio ./libraries/BluetoothHCI \
-           ./libraries/BluetoothHIDMaster ./libraries/NetBIOS ./libraries/Ticker; do
+           ./libraries/BluetoothHIDMaster ./libraries/NetBIOS ./libraries/Ticker \
+           ./libraries/VFS; do
     find $dir -type f \( -name "*.c" -o -name "*.h" -o -name "*.cpp" \) -a  \! -path '*api*' -exec astyle --suffix=none --options=./tests/astyle_core.conf \{\} \;
     find $dir -type f -name "*.ino" -exec astyle --suffix=none --options=./tests/astyle_examples.conf \{\} \;
 done