masterUncollatedFileOperation.H

Go to the documentation of this file.

/*---------------------------------------------------------------------------*\
  =========                 |
  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   \\    /   O peration     |
    \\  /    A nd           | www.openfoam.com
     \\/     M anipulation  |
-------------------------------------------------------------------------------
    Copyright (C) 2017 OpenFOAM Foundation
    Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
    This file is part of OpenFOAM.
 
    OpenFOAM is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
 
    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    for more details.
 
    You should have received a copy of the GNU General Public License
    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
Class
    Foam::fileOperations::masterUncollatedFileOperation
 
Description
    fileOperations that performs all file operations on the master processor.
    Requires the calls to be parallel synchronised!
 
    Limitations
    - no /processor in filename
    - no /uniform/ in the filename
 
    The main logic is in ::filePath which returns a
    - same path on all processors. This can either be a global file
      (system/controlDict, processorXXX/0/uniform/) or a collated file
      (processors/0/p)
    - same path on all processors of the local communicator
      (processors4_0-1/0/p)
    - different path on all processors (processor0/0/p)
 
    system/controlDict:
    filePath worldmaster: <globalRoot>/system/controlDict
             localmaster:           ,,
             slave      :           ,,
 
    processor0/uniform/time
    filePath worldmaster: <globalRoot>/processorXXX/uniform/time
             localmaster:           ,,
             slave      :           ,,
 
    processors0/0/p
    processors10/0/p
    processors10_2-4/0/p
 
\*---------------------------------------------------------------------------*/
 
#ifndef Foam_fileOperations_masterUncollatedFileOperation_H
#define Foam_fileOperations_masterUncollatedFileOperation_H
 
#include "fileOperation.H"
#include "OSspecific.H"
#include "HashPtrTable.H"
#include "DynamicList.H"
#include "List.H"
#include "unthreadedInitialise.H"
 
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
namespace Foam
{
 
// Forward Declarations
class PstreamBuffers;
 
namespace fileOperations
{
 
/*---------------------------------------------------------------------------*\
                     Class masterUncollatedFileOperation Declaration
\*---------------------------------------------------------------------------*/
 
class masterUncollatedFileOperation
:
    public fileOperation
{
protected:
 
    // Protected Data
 
        //- Any communicator allocated by me
        const label myComm_;
 
        //- Cached times for a given directory
        mutable HashPtrTable<DynamicList<instant>> times_;
 
 
    // Protected Operation Functors
 
        class mkDirOp
        {
            const mode_t mode_;
        public:
            mkDirOp(const mode_t mode)
            :
                mode_(mode)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::mkDir(f, mode_);
            }
        };
 
        class chModOp
        {
            const mode_t mode_;
        public:
            chModOp(const mode_t mode)
            :
                mode_(mode)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::chMod(f, mode_);
            }
        };
 
        class modeOp
        {
            const bool followLink_;
        public:
            modeOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            mode_t operator()(const fileName& f) const
            {
                return Foam::mode(f, followLink_);
            }
        };
 
        class typeOp
        {
            const bool followLink_;
        public:
            typeOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            // Returns label not fileName::Type for reductions
            label operator()(const fileName& f) const
            {
                return Foam::type(f, followLink_);
            }
        };
 
        class existsOp
        {
            const bool checkGzip_;
            const bool followLink_;
        public:
            existsOp(const bool checkGzip, const bool followLink)
            :
                checkGzip_(checkGzip),
                followLink_(followLink)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::exists(f, checkGzip_, followLink_);
            }
        };
 
        class isDirOp
        {
            const bool followLink_;
        public:
            isDirOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::isDir(f, followLink_);
            }
        };
 
        class isFileOp
        {
            const bool checkGzip_;
            const bool followLink_;
        public:
            isFileOp(const bool checkGzip, const bool followLink)
            :
                checkGzip_(checkGzip),
                followLink_(followLink)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::isFile(f, checkGzip_, followLink_);
            }
        };
 
        class fileSizeOp
        {
            const bool followLink_;
        public:
            fileSizeOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            off_t operator()(const fileName& f) const
            {
                return Foam::fileSize(f, followLink_);
            }
        };
 
        class lastModifiedOp
        {
            const bool followLink_;
        public:
            lastModifiedOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            time_t operator()(const fileName& f) const
            {
                return Foam::lastModified(f, followLink_);
            }
        };
 
        class highResLastModifiedOp
        {
            const bool followLink_;
        public:
            highResLastModifiedOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            double operator()(const fileName& f) const
            {
                return Foam::highResLastModified(f, followLink_);
            }
        };
 
        class mvBakOp
        {
            std::string ext_;
        public:
            mvBakOp(const std::string& ext)
            :
                ext_(ext)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::mvBak(f, ext_);
            }
        };
 
        class rmOp
        {
        public:
            bool operator()(const fileName& f) const
            {
                return Foam::rm(f);
            }
        };
 
        class rmDirOp
        {
            bool silent_;
        public:
            rmDirOp(const bool silent=false)
            :
                silent_(silent)
            {}
 
            bool operator()(const fileName& f) const
            {
                return Foam::rmDir(f, silent_);
            }
        };
 
        class cpOp
        {
            const bool followLink_;
        public:
            cpOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            bool operator()(const fileName& src, const fileName& dest) const
            {
                return Foam::cp(src, dest, followLink_);
            }
        };
 
        class lnOp
        {
        public:
            bool operator()(const fileName& src, const fileName& dest) const
            {
                return Foam::ln(src, dest);
            }
        };
 
        class mvOp
        {
            const bool followLink_;
        public:
            mvOp(const bool followLink)
            :
                followLink_(followLink)
            {}
 
            bool operator()(const fileName& src, const fileName& dest) const
            {
                return Foam::mv(src, dest, followLink_);
            }
        };
 
        class fileOrNullOp
        {
            const bool isFile_;
        public:
            fileOrNullOp(const bool isFile)
            :
                isFile_(isFile)
            {}
 
            const fileName& operator()(const fileName& f) const
            {
                return
                (
                    (isFile_ ? Foam::isFile(f) : Foam::isDir(f))
                  ? f
                  : fileName::null
                );
            }
        };
 
        class readDirOp
        {
            const fileName::Type type_;
            const bool filtergz_;
            const bool followLink_;
        public:
            readDirOp
            (
                const fileName::Type type,
                const bool filtergz,
                const bool followLink
            )
            :
                type_(type),
                filtergz_(filtergz),
                followLink_(followLink)
            {}
 
            fileNameList operator()(const fileName& f) const
            {
                return Foam::readDir(f, type_, filtergz_, followLink_);
            }
        };
 
 
    // Private Member Functions
 
        //- Get the list of processors that are part of this communicator
        static labelList subRanks(const label n);
 
        template<class Type>
        Type scatterList(const UList<Type>&, const int, const label comm) const;
 
        template<class Type, class FileOp>
        Type masterOp
        (
            const fileName& fName,
            const FileOp& fop,
            const int tag,
            const label comm
        ) const;
 
        template<class Type, class FileOp>
        Type masterOp
        (
            const fileName& src,
            const fileName& dest,
            const FileOp& fop,
            const int tag,
            const label comm
        ) const;
 
        //- Equivalent of Time::findInstance
        static word findInstancePath
        (
            const instantList& timeDirs,
            const instant& t
        );
 
        //- Search (locally!) for object; return info on how it was found.
        //  Does not do any parallel communication.
        //    checkGlobal : also check undecomposed case
        //    isFile      : true:check for file  false:check for directory
        // searchType    : how was found
        // processorsDir : name of processor directory
        // instance      : instance
        virtual fileName filePathInfo
        (
            const bool checkGlobal,
            const bool isFile,
            const IOobject&,
            const bool search,
            pathType& searchType,
            word& processorsDir,
            word& instance
        ) const;
 
        //- Construct filePath
        fileName localObjectPath
        (
            const IOobject&,
            const pathType& searchType,
            const word& processorsDir,
            const word& instancePath
        ) const;
 
        //- Read file contents and send to processors.
        //  Handles compressed or uncompressed files
        static void readAndSend
        (
            const fileName& filePath,
            const labelUList& procs,
            PstreamBuffers& pBufs
        );
 
        //- Read files on comms master
        static autoPtr<ISstream> read
        (
            IOobject& io,
            const label comm,
            const bool uniform,             // on comms master only
            const fileNameList& filePaths,  // on comms master only
            const boolList& procValid       // on comms master only
        );
 
        //- Helper: check IO for local existence. Like filePathInfo but
        //  without parent searchign and instance searching
        bool exists(const dirIndexList&, IOobject& io) const;
 
 
public:
 
        //- Runtime type information
        TypeName("masterUncollated");
 
 
    // Static Data
 
        //- Max size of parallel communications. Switches from non-blocking
        //  to scheduled when reading/writing files. Read as float to enable
        //  easy specification of large sizes.
        static float maxMasterFileBufferSize;
 
 
    // Constructors
 
        //- Default construct
        explicit masterUncollatedFileOperation(bool verbose);
 
        //- Construct from communicator
        masterUncollatedFileOperation(const label comm, bool verbose);
 
 
    //- Destructor
    virtual ~masterUncollatedFileOperation();
 
 
    // Member Functions
 
        // OSSpecific equivalents
 
            //- Make directory
            virtual bool mkDir(const fileName&, mode_t=0777) const;
 
            //- Set the file mode
            virtual bool chMod(const fileName&, const mode_t) const;
 
            //- Return the file mode
            virtual mode_t mode
            (
                const fileName&,
                const bool followLink = true
            ) const;
 
            //- Return the file type: DIRECTORY, FILE or SYMLINK
            virtual fileName::Type type
            (
                const fileName&,
                const bool followLink = true
            ) const;
 
            //- Does the name exist (as DIRECTORY or FILE) in the file system?
            //  Optionally enable/disable check for gzip file.
            virtual bool exists
            (
                const fileName&,
                const bool checkGzip=true,
                const bool followLink = true
            ) const;
 
            //- Does the name exist as a DIRECTORY in the file system?
            virtual bool isDir
            (
                const fileName&,
                const bool followLink = true
            ) const;
 
            //- Does the name exist as a FILE in the file system?
            //  Optionally enable/disable check for gzip file.
            virtual bool isFile
            (
                const fileName&,
                const bool checkGzip=true,
                const bool followLink = true
            ) const;
 
            //- Return size of file
            virtual off_t fileSize
            (
                const fileName&,
                const bool followLink = true
            ) const;
 
            //- Return time of last file modification
            virtual time_t lastModified
            (
                const fileName&,
                const bool followLink = true
            ) const;
 
            //- Return time of last file modification
            virtual double highResLastModified
            (
                const fileName&,
                const bool followLink = true
            ) const;
 
            //- Read a directory and return the entries as a string list
            virtual fileNameList readDir
            (
                const fileName&,
                const fileName::Type=fileName::FILE,
                const bool filtergz=true,
                const bool followLink = true
            ) const;
 
            //- Copy, recursively if necessary, the source to the destination
            virtual bool cp
            (
                const fileName& src,
                const fileName& dst,
                const bool followLink = true
            ) const;
 
            //- Create a softlink. dst should not exist. Returns true if
            //  successful.
            virtual bool ln(const fileName& src, const fileName& dst) const;
 
            //- Rename src to dst
            virtual bool mv
            (
                const fileName& src,
                const fileName& dst,
                const bool followLink = false
            ) const;
 
            //- Rename to a corresponding backup file
            //  If the backup file already exists, attempt with
            //  "01" .. "99" suffix
            virtual bool mvBak
            (
                const fileName&,
                const std::string& ext = "bak"
            ) const;
 
            //- Remove a file, returning true if successful otherwise false
            virtual bool rm(const fileName&) const;
 
            //- Remove a directory and its contents
            //  \param silent do not report missing directory
            virtual bool rmDir
            (
                const fileName& dir,
                const bool silent = false
            ) const;
 
//            //- Open a shared library. Return handle to library. Print error
//            //  message if library cannot be loaded (check = true)
//            virtual void* dlOpen
//            (
//                const fileName& lib,
//                const bool check = true
//            ) const;
 
 
        // (reg)IOobject functinality
 
            //- Search for an object. checkGlobal : also check undecomposed case
            virtual fileName filePath
            (
                const bool checkGlobal,
                const IOobject& io,
                const word& typeName,
                const bool search
            ) const;
 
            //- Search for a directory. checkGlobal : also check undecomposed
            //  case
            virtual fileName dirPath
            (
                const bool checkGlobal,
                const IOobject& io,
                const bool search
            ) const;
 
            //- Search directory for objects. Used in IOobjectList.
            virtual fileNameList readObjects
            (
                const objectRegistry& db,
                const fileName& instance,
                const fileName& local,
                word& newInstance
            ) const;
 
            //- Read object header from supplied file
            virtual bool readHeader
            (
                IOobject&,
                const fileName&,
                const word& typeName
            ) const;
 
            //- Reads header for regIOobject and returns an ISstream
            //  to read the contents.
            virtual autoPtr<ISstream> readStream
            (
                regIOobject&,
                const fileName&,
                const word& typeName,
                const bool valid = true
            ) const;
 
            //- Top-level read
            virtual bool read
            (
                regIOobject&,
                const bool masterOnly,
                const IOstreamOption::streamFormat format,
                const word& typeName
            ) const;
 
            //- Writes a regIOobject (so header, contents and divider).
            //  Returns success state.
            virtual bool writeObject
            (
                const regIOobject& io,
                IOstreamOption streamOpt = IOstreamOption(),
                const bool valid = true
            ) const;
 
            //- Generate an ISstream that reads a file
            virtual autoPtr<ISstream> NewIFstream(const fileName&) const;
 
            //- Generate an OSstream that writes a file
            virtual autoPtr<OSstream> NewOFstream
            (
                const fileName& pathname,
                IOstreamOption streamOpt = IOstreamOption(),
                const bool valid = true
            ) const;
 
 
        // File modification checking
 
            //- Add watching of a file. Returns handle
            virtual label addWatch(const fileName&) const;
 
            //- Remove watch on a file (using handle)
            virtual bool removeWatch(const label) const;
 
            //- Find index (or -1) of file in list of handles
            virtual label findWatch
            (
                const labelList& watchIndices,
                const fileName&
            ) const;
 
            //- Helper: add watches for list of regIOobjects
            virtual void addWatches(regIOobject&, const fileNameList&) const;
 
            //- Get name of file being watched (using handle)
            virtual fileName getFile(const label) const;
 
            //- Update state of all files
            virtual void updateStates
            (
                const bool masterOnly,
                const bool syncPar
            ) const;
 
            //- Get current state of file (using handle)
            virtual fileMonitor::fileState getState(const label) const;
 
            //- Set current state of file (using handle) to unmodified
            virtual void setUnmodified(const label) const;
 
 
        // Other
 
            //- Same file?
            static bool uniformFile(const fileNameList&);
 
            //- Get sorted list of times
            virtual instantList findTimes(const fileName&, const word&) const;
 
            //- Find instance where IOobject is.
            //  FatalError if it cannot be found and readOpt is
            //  (MUST_READ or MUST_READ_IF_MODIFIED).
            //  Otherwise it returns the stopInstance.
            virtual IOobject findInstance
            (
                const IOobject& io,
                const scalar startValue,
                const word& stopInstance
            ) const;
 
            //- Callback for time change
            virtual void setTime(const Time&) const;
 
            //- Forcibly wait until all output done. Flush any cached data
            virtual void flush() const;
 
            //- Return cached times
            const HashPtrTable<DynamicList<instant>>& times() const noexcept
            {
                return times_;
            }
};
 
 
/*---------------------------------------------------------------------------*\
           Class masterUncollatedFileOperationInitialise Declaration
\*---------------------------------------------------------------------------*/
 
class masterUncollatedFileOperationInitialise
:
    public unthreadedInitialise
{
public:
 
    // Constructors
 
        //- Construct from components
        masterUncollatedFileOperationInitialise(int& argc, char**& argv)
        :
            unthreadedInitialise(argc, argv)
        {}
 
 
    //- Destructor
    virtual ~masterUncollatedFileOperationInitialise() = default;
};
 
 
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
} // End namespace fileOperations
} // End namespace Foam
 
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
#ifdef NoRepository
    #include "masterUncollatedFileOperationTemplates.C"
#endif
 
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
#endif
 
// ************************************************************************* //