Announcement

Collapse
No announcement yet.

Cleaning up IFS subfolders

Collapse
X
 
  • Filter
  • Time
  • Show
Clear All
new posts

  • Cleaning up IFS subfolders

    I am storing some stream files in a folder in the IFS. The name of the each subfolder is a timestamp. Example,

    /home/PROD/2020-12-08-08.08.15.087000
    /home/PROD/2020-12-08-08.08.41.631000
    /home/PROD/2020-12-08-08.09.07.481000
    .
    .
    .

    I want to perform cleanup every month by, ideally, dumping the folder names to an outfile and then reading the outfile and if the first 10 characters of the subfolder name equates to a date that is more than 30 days prior to todays date, delete the subfolder.

    Any suggestions how to do this or something similar to accomplish the same thing

  • #2
    IMHO, its easier, faster, and more robust to just read the filenames and evaluate the dates from them. Here's a quick example (I didn't really test this, but it is cobbled together from parts of other working programs):

    Code:
    **free
    ctl-opt dftactgrp(*no) actgrp(*new)
            option(*srcstmt:*nodebugio:*noshowcpy);
    
    /copy ifsio_h
    
    dcl-c PARENT_DIR '/home/PROD';
    
    dcl-s  PurgeDate date;
    dcl-s  FileDate  date;
    dcl-s  dirh      pointer;
    dcl-s  p_ent     pointer;
    dcl-ds ent       likeds(dirent) based(p_ent);
    dcl-s  filename  varchar(5000);
    dcl-s  first10   char(10);
    
    PurgeDate = %date() - %days(30);
    
    dirh = opendir(%trimr(PARENT_DIR));
    if dirh = *null;
      ReportError();
      return;
    endif;
    
    p_ent = readdir(dirh);
    dow p_ent <> *null;
    
      filename = %str(%addr(ent.d_name));
      if %len(filename) >= 10;
        first10 = %subst(filename:1:10);
        test(de) first10;
        if not %error;
          FileDate = %date(first10);
          if FileDate <= PurgeDate;
            deleteDirectory(PARENT_DIR + '/' + filename);
          endif;
        endif;
      endif;
    
      p_ent = readdir(dirh);
    enddo;
    
    closedir(dirh);
    
    *inlr = *on;
    return;
    
    
    // *=================================================================
    // *  The IFS APIs do not provide a command that deletes all objects
    // *  within a directory. We could write a routine to do that, but,
    // *  its easier to just call the CL command.
    // *=================================================================
    dcl-proc deleteDirectory;
    
      dcl-pi *N;
        pathName varchar(5000) const;
      end-pi;
    
      dcl-pr QCMDEXC extpgm;
        command char(32767)   const options(*varsize);
        len     packed(15: 5) const;
        igc     char(3)       const options(*nopass);
      end-pr;
    
      dcl-s cmd varchar(5100);
    
      cmd = 'RMVDIR DIR(''' + %trim(pathname) + ''') +
                    SUBTREE(*ALL)';
      QCMDEXC(cmd: %len(cmd));
    
    end-proc;
    
    
    // *=================================================================
    // *  ReportError():  Look up the last error number that occurred
    // *                  and send an exception message with that error
    // *                  number.
    // *
    // *  NOTE: This terminates the program unless you MONITOR for it.
    // *=================================================================
    dcl-proc ReportError;
    
      dcl-pr QMHSNDPM extpgm;
        MessageID  char(7)     const;
        QualMsgF   char(20)    const;
        MsgData    char(32767) const options(*varsize);
        MsgDtaLen  int(10)     const;
        MsgType    char(10)    const;
        CallStkEnt char(10)    const;
        CallStkCnt int(10)     const;
        MessageKey char(4);
        ErrorCode  char(32767) options(*varsize);
      end-pr;
    
      dcl-ds FailWithException;
        *n int(10) inz(0);
        *n int(10) inz(0);
      end-ds;
    
      dcl-pr get_errno pointer extproc('__errno');
      end-pr;
    
      dcl-s p_errno pointer;
      dcl-s errno   int(10)  based(p_errno);
      dcl-s MsgKey  char(4);
      dcl-s MsgId   char(7);
    
      p_errno = get_errno();
      MsgID = 'CPE' + %editc(%dec(errno:4:0): 'X');
    
      QMHSNDPM( MsgID
              : 'QCPFMSG   *LIBL'
              : ' '
              : 0
              : '*ESCAPE'
              : '*PGMBDY'
              : 1
              : MsgKey
              : FailWithException );
    
    end-proc;
    If you need the IFSIO_H copybook, you can get it here:
    http://www.scottklement.com/rpg/copy...io_h.rpgle.txt

    Comment


    • #3
      This could potentially be quite simple if you can work with the file attributes instead of the file name. Does the timestamp in the directory name roughly match the time you last created or removed a file in the directory?

      If so, you can use a single "find" command in QShell to identify these and perform an action on them (in this case, removing them).

      https://www.ibm.com/support/knowledg.../rzahzfind.htm

      Something like this:

      Code:
      find /home/PROD/ -type d -mtime +31 -exec rm -rf {} \;
      Breaking this down:
      • Code:
        -type d
        says to only look for directories
      • Code:
        -mtime +31
        says to check the directory's modified time, which will correspond with when the contents of the directory last changed. Items will match where it exceeds 31 days.
      • Code:
        -exec rm -rf {} \;
        says "run the 'rm' command for each matching result, replacing {} with the path found". 'rm', in this case, is the command to remove files and directories.
      Please be very careful running this! If you run this over the wrong directory, you could end up deleting a lot of stuff. If you would like to test what would be matched by the find command, you take off the -exec section of the command. This will instead output the matching results to the screen.

      If you find a command string that works for you, you could then invoke it with the QSH command and put it on the job scheduler, for example.

      Comment


      • #4
        There is no need to generate a tempoaray file!
        Have a look at the IFS_OBJECT_STATISTICS table function service:
        https://www.ibm.com/support/knowledg...ifsobjstat.htm

        Code:
        SELECT x.*
            FROM TABLE (QSYS2.IFS_OBJECT_STATISTICS(START_PATH_NAME => '/home/PROD',
                                                    SUBTREE_DIRECTORIES => 'YES')) x
           Where     Object_Type = '*DIR'
                 and Date(Substr(Path_Name, Locate_In_String(IFSFile, '/', -1) + 1, 10)) < Current_Date - 30 Days
        Birgitta
        Last edited by B.Hauser; December 15, 2020, 09:56 AM.

        Comment


        • #5
          Originally posted by B.Hauser View Post
          There is no need to generate a tempoaray file!
          Have a look at the IFS_OBJECT_STATISTICS table function service:
          https://www.ibm.com/support/knowledg...ifsobjstat.htm

          Code:
          SELECT x.*
          FROM TABLE (QSYS2.IFS_OBJECT_STATISTICS(START_PATH_NAME => '/home/PROD',
          SUBTREE_DIRECTORIES => 'YES')) x
          Where Object_Type = '*DIR'
          and Date(Substr(Path_Name, Locate_In_String(IFSFile, '/', -1) + 1, 10)) < Current_Date - 30 Days
          Birgitta
          I'm getting 'IFS_OBJECT_STATISTICS in QSYS2 type *N not found'

          IFS_OBJECT_STATISTICS in QSYS2 type *N not found.
          SELECT x.* FROM TABLE (QSYS2.IFS_OBJECT_STATISTICS(START_PATH_NAME
          => '/home/PROD', SUBTREE_DIRECTORIES => 'YES')) x Where Object_Type
          = '*DIR' and Path_Name between
          '/home/PROD/2020-01-01-00.00.00.000000' and
          '/home/PROD/2020-12-30-24.00.00.000000'


          The 3 subfolders I listed in the OP do exist in /home/PROD

          Comment


          • #6
            Originally posted by Scott Klement View Post
            IMHO, its easier, faster, and more robust to just read the filenames and evaluate the dates from them. Here's a quick example (I didn't really test this, but it is cobbled together from parts of other working programs):
            If you need the IFSIO_H copybook, you can get it here:
            http://www.scottklement.com/rpg/copy...io_h.rpgle.txt
            This seems to work quite well, thanks, Scott. Oddly enough, in using the *free option to start coding from column one, Rdi does not display the variable values when hovering over them in debug. When I shift the code to the right and remove the *free option, all is well. Maybe there is a setting I need to activate. Not a big deal.........thanks again

            Comment


            • #7
              Originally posted by gregwga50 View Post

              I'm getting 'IFS_OBJECT_STATISTICS in QSYS2 type *N not found'

              IFS_OBJECT_STATISTICS in QSYS2 type *N not found.
              SELECT x.* FROM TABLE (QSYS2.IFS_OBJECT_STATISTICS(START_PATH_NAME
              => '/home/PROD', SUBTREE_DIRECTORIES => 'YES')) x Where Object_Type
              = '*DIR' and Path_Name between
              '/home/PROD/2020-01-01-00.00.00.000000' and
              '/home/PROD/2020-12-30-24.00.00.000000'


              The 3 subfolders I listed in the OP do exist in /home/PROD
              Is V7R4 required for this? If so, that is my problem, we are on V7R3

              Comment


              • jtaylor___
                jtaylor___ commented
                Editing a comment
                It works for me on 7.3. Are you current of PTFs?

            • #8
              IFS_OBJECT_STATISTICS should work with i7.3 TR7.

              If you're not there, you should catch up on these technology releases because there's lots of good stuff in new system functionality, RPG enhancements, and SQL enhancements.

              We just went to i7.3 TR9 this past weekend.

              Comment


              • #9
                Greg, you mentioned RDi in your post above. Which version are you on? The newest is 9.6.0.9, which I just upgraded to this past weekend as well. Perhaps that would fix the issue you're experiencing.

                Comment


                • #10
                  I would recommend you to take a look at Simon Hutchinsons homepage
                  http://rpgpgm.com

                  He has many excellent articles.
                  Among those is one called "Using SQL to list directories and files in IFS"
                  https://www.rpgpgm.com/2019/11/using...and-files.html

                  This might be just what you need.

                  Comment


                  • #11
                    Just my 2 cents with a very basic program we've been using for years. We roll through a file which contains the path and number of days to keep files. We use CGIDEV2 and Scott's UDTF IFSDIR to accomplish this.


                    Code:
                    Field Pos Pos Len/Dec/Typ Text
                    IFSDELR 55 Item Nu
                    IPATH 1 50 50 A
                    IDAYS 51 55 5 A

                    Code:
                    fIFSDELPF if e k disk
                    
                    d PurgeDirectory pr extpgm( 'IFSDIRDEL' )
                    d dir 50a const
                    d numberDays 5a const
                    
                    /free
                    
                    setll *start IFSDELPF;
                    read IFSDELPF;
                    dow not %eof( IFSDELPF );
                    PurgeDirectory( ipath : idays );
                    read IFSDELPF;
                    enddo;
                    
                    *inlr = *on;
                    
                    /end-free


                    Code:
                    
                    h bnddir( 'CGIDEV2' )
                    h dftactgrp( *no )
                    
                    /copy cgidev2/qrpglesrc,hspecs
                    /copy cgidev2/qrpglesrc,hspecsbnd
                    
                    /copy cgidev2/qrpglesrc,prototypeb
                    /copy cgidev2/qrpglesrc,usec
                    
                    *this is the *entry plist ............................................
                    d IFSDIRDEL pr extpgm( 'IFSDIRDEL' )
                    d dir 50a const
                    d numberDays 5a const
                    
                    d IFSDIRDEL pi
                    d dir 50a const
                    d numberDays 5a const
                    *................................................. ....................
                    
                    d*doc s 500a varying
                    d fileName s 150a
                    d n s 5i 0
                    d path s 50a varying
                    d*type s 10a
                    
                    /free
                    
                    path = %trim( dir );
                    n = %int( numberDays );
                    
                    exec sql set option naming = *SYS ;
                    
                    exec sql declare c1 cursor for
                    select filename from table( qgpl/ifsdir( :path ) ) as t
                    
                    where ( type = '*STMF' )
                    and ( modify_time < current timestamp - :n days )
                    
                    order by modify_time ;
                    
                    exec sql open c1;
                    
                    exec sql fetch c1 into :filename ;
                    
                    dow sqlcod = 0;
                    
                    unlink( %trim( path ) + '/' + %trimr( filename ) );
                    
                    exec sql fetch next from c1 into :filename ;
                    
                    enddo;
                    
                    exec sql close c1;
                    
                    *inlr = *on;
                    
                    /end-free
                    Your friends list is empty!

                    Comment

                    Working...
                    X