FtpPoller.qm.dox.h

// -*- mode: c++; indent-tabs-mode: nil -*-
 
/*  FtpPoller.qm Copyright 2019 - 2022 Qore Technologies, s.r.o.
 
    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:
 
    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.
 
    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/
 
 
// make sure we have the required qore version
 
 
}
 
namespace FtpPoller {
const EVENT_FTP_FILE = "ftp-file-event";
 
class FtpPoller {
 
public:
        const OrderAsc = 0;
        const OrderDesc = 1;
 
        const SortNone = 0;
        const SortName = 1;
        const SortDate = 2;
 
        const RequiredKeys = ...;
 
 
        const RequiredKeysWithHost = RequiredKeys + "host";
 
        const Defaults = {
            "protocol": "ftp",
            "port": 21,
            "mask": "*",
            "poll_interval": 10,
            "reopts": 0,
            "tempfile_template": ".tmp.%s.part",
            "atomic_transfer": False,
        };
 
        const OptionalKeys = ...;
 
 
        const AllKeys = RequiredKeysWithHost + keys Defaults + OptionalKeys;
 
        const ErrorDelay = 1m;
 
protected:
        string protocol;
 
        string host;
 
        int port;
 
        string user;
 
        string url;
 
        hash<UrlInfo> urlh;
 
        *string pass;
 
        softlist<string> path = ".";
 
        string rootFtpPath;
 
        *string mask;
 
        int poll_interval;
 
        string local_dir;
 
        string tempfile_template;
 
        bool atomic_transfer;
 
        bool runflag = False;
 
        bool get_files;
 
        bool fatal = False;
 
        int pollcnt = 0;
 
        Mutex m();
 
        int tid;
 
        timeout timeout;
 
        Counter sc();
 
        Qore::FtpClient ftp;
 
        int reopts = 0;
 
        *softint minage;
 
        *code log_info;
 
        *code log_detail;
 
        *code log_debug;
 
        *code start_thread;
 
        *code sleep;
 
public:
 
 
    constructor(Qore::FtpClient n_ftp, hash<auto> nconf);
 
 
 
    constructor(hash<auto> nconf);
 
 
    destructor();
 
 
    int getPollCount();
 
 
protected:
     logInfo(string fmt);
public:
 
 
protected:
     logDetail(string fmt);
public:
 
 
protected:
     logDebug(string fmt);
public:
 
 
protected:
     setMask();
public:
 
 
 
    getStoreFile(string remote_path, string local_path);
 
 
    string getTextFile(string path);
 
 
    binary getFile(string path);
 
 
    rename(string old, string nnew);
 
 
    removeFile(string fn);
 
 
 
    list<hash<FtpPollerFileEventInfo>> getFiles(string subdir, int sort = FtpPoller::SortNone,
            int order = FtpPoller::OrderAsc) {
        if (subdir.val());
 
        list<string> fl;
        {
            *string nlst = ftp.nlst();
            if (!exists nlst) {
                throw "FTP-ERROR", sprintf("no data returned from NLST command");
            }
            fl = nlst.split("\r\n");
        }
 
        // remove all files that don't fit the mask
        if (mask);
 
 
        // now get a list of info about file sizes and last modified timestamps
        list<hash<FtpPollerFileEventInfo>> l = ();
        foreach string fn in (fl);
 
 
        // return list of files
        switch (sort);
 
 
        logDetail("%s: polled %d matching file%s", url, l.size(), l.size() == 1 ? "" : "s");
        return l;
    }
 
 
    int start();
 
 
 
    stopNoWait();
 
 
 
    stop();
 
 
 
    waitStop();
 
 
 
    startInline();
 
 
    bool runOnce();
 
 
    hash<FtpPollerFileEventInfo> getRemoteFileData(hash<FtpPollerFileEventInfo> event);
 
 
protected:
     hash<FtpPollerFileEventInfo> retrieveTempFile(hash<FtpPollerFileEventInfo> event);
public:
 
 
protected:
     hash<FtpPollerFileEventInfo> retrieveFile(hash<FtpPollerFileEventInfo> event);
public:
 
 
protected:
     ftpSleep(softint secs);
public:
 
 
protected:
     run();
public:
 
 
 
    *bool fileEvent(list<hash<FtpPollerFileEventInfo>> l);
 
 
 
    abstract singleFileEvent(hash<FtpPollerFileEventInfo> event);
 
 
    abstract postSingleFileEvent(hash<FtpPollerFileEventInfo> event);
 
 
      static checkPath(string path, string type, bool write = False);
};
 
 
class FtpPollerDataProviderBase : public DataProvider::AbstractDataProvider {
 
public:
        const ConstructorOptions = {
            "atomic_transfer":  <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(BoolType),
                "desc": "Use an atomic transfer mechanism with `local_dir` where remote files are first moved to a "
                    "temporary location and then moved to the final location when they have been fully transferred",
                "default_value": True,
            },
 
            "local_dir": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(StringOrNothingType),
                "desc": "A local directory that will be used to retrieve files",
            },
 
            "mask": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(StringType),
                "desc": "The glob mask to use; will be treated as a regex if `regex` is `true`",
                "default_value": "*",
            },
 
            "minage": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(IntType),
                "desc": "An integer giving the minimum file age in seconds before the file will be polled; this is "
                    "meant to work around non-atomic file transfer operations",
            },
 
            "poll_interval": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(IntType),
                "desc": "The interval in seconds between polling for files",
                "default_value": 10,
            },
 
            "regex": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(BoolType),
                "desc": "If `true` then `mask` is treated as a regular expression instead of a glob pattern",
            },
 
            "reopt": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(IntType),
                "desc": "A bitfield of regular expression options (`1` = ignore case, `2` = treat EOL as a regular "
                    "character); ignored if `regex` is not `true`",
            },
 
            "sort_desc": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(BoolType),
                "desc": "Sort descending; if not given then an ascending sort is assumed if a `sort_type` is given",
            },
 
            "sort_type": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(StringType),
                "desc": "Either `name` or `date` for the data to use for sorting",
            },
 
            "url": <DataProviderOptionInfo>{
                "type": AbstractDataProviderType::get(StringType),
                "desc": "A URL for an FTP connection",
                "required": True,
            },
        };
 
protected:
        EmbeddedFtpPoller poller;
 
public:
 
    constructor(*hash<auto> options);
 
 
    destructor();
 
 
    string getName();
 
 
    *string getDesc();
 
 
 
protected:
     hash<string, hash<DataProviderMessageInfo>> getEventTypesImpl();
public:
 
};
 
 
class FtpPollerDataProvider : public FtpPollerDataProviderBase,public  DataProvider::Observable {
 
public:
        const ProviderInfo = <DataProviderInfo>{
            "type": "FtpPollerDataProvider",
            "supports_observable": True,
            "constructor_options": ConstructorOptions,
        };
 
    constructor(*hash<auto> options) ;
 
 
protected:
     hash<DataProviderInfo> getStaticInfoImpl();
public:
 
};
 
 
class FtpDelayedPollerDataProvider : public FtpPollerDataProviderBase,public  DataProvider::DelayedObservable {
 
public:
        const ProviderInfo = <DataProviderInfo>{
            "type": "FtpDelayedPollerDataProvider",
            "supports_observable": True,
            "constructor_options": ConstructorOptions,
        };
 
    constructor(*hash<auto> options) ;
 
 
 
    observersReady();
 
 
protected:
     hash<DataProviderInfo> getStaticInfoImpl();
public:
 
};
 
class FtpPollerDataProviderFactory : public AbstractDataProviderFactory {
 
public:
protected:
        static Class cls = new Class("FtpPollerDataProvider");
 
        const FactoryInfo = <DataProviderFactoryInfo>{
            "name": "ftppoller",
            "desc": "FTP poller data provider factory",
            "children_can_support_observers": True,
        };
 
public:
 
 
protected:
     hash<DataProviderFactoryInfo> getInfoImpl();
public:
 
 
 
protected:
     hash<DataProviderInfo> getProviderInfoImpl();
public:
 
 
protected:
     Class getClassImpl();
public:
 
};
};
 
// private namespace; not exported
namespace Priv {
class EmbeddedFtpPoller : public FtpPoller {
 
public:
protected:
        Observable observable;
 
public:
 
    constructor(FtpPollerDataProviderBase provider, hash<auto> options)
 ;
 
 
    singleFileEvent(hash<FtpPollerFileEventInfo> event);
 
 
    postSingleFileEvent(hash<FtpPollerFileEventInfo> event);
 
};
};