001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.net.ftp.parser; 019 020import java.text.ParseException; 021import java.util.List; 022 023import org.apache.commons.net.ftp.Configurable; 024import org.apache.commons.net.ftp.FTPClientConfig; 025import org.apache.commons.net.ftp.FTPFile; 026import org.apache.commons.net.ftp.FTPFileEntryParser; 027 028/** 029 * Implements {@link FTPFileEntryParser} and {@link Configurable} for IBM zOS/MVS Systems. 030 * 031 * @see FTPFileEntryParser Usage instructions. 032 */ 033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 034 035 static final int UNKNOWN_LIST_TYPE = -1; 036 static final int FILE_LIST_TYPE = 0; 037 static final int MEMBER_LIST_TYPE = 1; 038 static final int UNIX_LIST_TYPE = 2; 039 static final int JES_LEVEL_1_LIST_TYPE = 3; 040 static final int JES_LEVEL_2_LIST_TYPE = 4; 041 042 /** 043 * Dates are ignored for file lists, but are used for member lists where possible 044 */ 045 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 046 // 13:52 047 048 /** 049 * Matches these entries: 050 * 051 * <pre> 052 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 053 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 054 * </pre> 055 * 056 * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a> 057 */ 058 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 059 // ignored 060 "\\S+\\s+" + // unit - ignored 061 "\\S+\\s+" + // access date - ignored 062 "\\S+\\s+" + // extents -ignored 063 // If the values are too large, the fields may be merged (NET-639) 064 "(?:\\S+\\s+)?" + // used - ignored 065 "\\S+\\s+" + // recfm - ignored 066 "\\S+\\s+" + // logical record length -ignored 067 "\\S+\\s+" + // block size - ignored 068 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist 069 // but only support: PS, PO, PO-E 070 "(\\S+)\\s*"; // Dataset Name (file name) 071 072 /** 073 * Matches these entries: 074 * 075 * <pre> 076 * Name VV.MM Created Changed Size Init Mod Id 077 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 078 * </pre> 079 */ 080 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 081 "\\S+\\s+" + // version, modification (ignored) 082 "\\S+\\s+" + // create date (ignored) 083 "(\\S+)\\s+" + // modification date 084 "(\\S+)\\s+" + // modification time 085 "\\S+\\s+" + // size in lines (ignored) 086 "\\S+\\s+" + // size in lines at creation(ignored) 087 "\\S+\\s+" + // lines modified (ignored) 088 "\\S+\\s*"; // id of user who modified (ignored) 089 090 /** 091 * Matches these entries, note: no header: 092 * 093 * <pre> 094 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 095 * 012345678901234567890123456789012345678901234 096 * 1 2 3 4 097 * </pre> 098 */ 099 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored 100 "(\\S+)\\s+" + // job number 101 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 102 "(\\S+)\\s+" + // number of spool files 103 "(\\S+)\\s+" + // Text "Spool" ignored 104 "(\\S+)\\s*" // Text "Files" ignored 105 ; 106 107 /** 108 * JES INTERFACE LEVEL 2 parser Matches these entries: 109 * 110 * <pre> 111 * JOBNAME JOBID OWNER STATUS CLASS 112 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 113 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 114 * </pre> 115 * 116 * Sample output from FTP session: 117 * 118 * <pre> 119 * ftp> quote site filetype=jes 120 * 200 SITE command was accepted 121 * ftp> ls 122 * 200 Port request OK. 123 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 124 * JOBNAME JOBID OWNER STATUS CLASS 125 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 126 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 127 * 250 List completed successfully. 128 * ftp> ls job01906 129 * 200 Port request OK. 130 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 131 * JOBNAME JOBID OWNER STATUS CLASS 132 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 133 * -------- 134 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT 135 * 001 JES2 A JESMSGLG 858 136 * 002 JES2 A JESJCL 128 137 * 003 JES2 A JESYSMSG 443 138 * 3 spool files 139 * 250 List completed successfully. 140 * </pre> 141 */ 142 143 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored 144 "(\\S+)\\s+" + // job number 145 "(\\S+)\\s+" + // owner ignored 146 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 147 "(\\S+)\\s+" + // job class ignored 148 "(\\S+).*" // rest ignored 149 ; 150 151 private int isType = UNKNOWN_LIST_TYPE; 152 153 /** 154 * Fallback parser for Unix-style listings 155 */ 156 private UnixFTPEntryParser unixFTPEntryParser; 157 158 /* 159 * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is 160 * the operating system on the mainframe, and is the new name for MVS) 161 * 162 * The file system on the mainframe does not have hierarchical structure as for example the unix file system. For a more comprehensive description, 163 * please refer to the IBM manuals 164 * 165 * @LINK: https://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 166 * 167 * 168 * Dataset names ============= 169 * 170 * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be 171 * max 44 characters including the dots. 172 * 173 * 174 * Dataset organisation ==================== 175 * 176 * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organisation 177 * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer. 178 * 179 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset 180 * 181 * The PS file is just a flat file, as you would find it on the unix file system. 182 * 183 * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It 184 * is possible to CD into the file, and to retrieve the individual members. 185 * 186 * 187 * Dataset record format ===================== 188 * 189 * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the 190 * FTP transfer. 191 * 192 * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of 193 * records per allocated storage, and V means a variable number of records. 194 * 195 * 196 * Other notes =========== 197 * 198 * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct 199 * Access Device KJ.IOP998.ERROR.PL.UNITTEST 200 * 201 * 202 * Implementation notes ==================== 203 * 204 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed. 205 * 206 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 207 * FTPFile.Timestamp: change time or null 208 * 209 * 210 * 211 * Additional information ====================== 212 * 213 * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site 214 * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job 215 * scheduler DB2 is used to interact with a DB2 subsystem 216 * 217 * This parser supports SEQ and JES. 218 */ 219 220 /** 221 * The sole constructor for a MVSFTPEntryParser object. 222 */ 223 public MVSFTPEntryParser() { 224 super(""); // note the regex is set in preParse. 225 super.configure(null); // configure parser with default configurations 226 } 227 228 /* 229 * @return 230 */ 231 @Override 232 protected FTPClientConfig getDefaultConfiguration() { 233 return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null); 234 } 235 236 /** 237 * Parses entries representing a dataset list. 238 * <pre> 239 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 240 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 241 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 242 * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 243 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU 244 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 245 * Migrated HLQ.DATASET.NAME 246 * </pre> 247 * <pre> 248 * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7] 249 * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name 250 * </pre> 251 * 252 * @param entry zosDirectoryEntry 253 * @return null: entry was not parsed. 254 */ 255 private FTPFile parseFileList(final String entry) { 256 if (matches(entry)) { 257 final FTPFile file = new FTPFile(); 258 file.setRawListing(entry); 259 final String name = group(2); 260 final String dsorg = group(1); 261 file.setName(name); 262 263 // DSORG 264 if ("PS".equals(dsorg)) { 265 file.setType(FTPFile.FILE_TYPE); 266 } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 267 // regex already ruled out anything other than PO or PO-E 268 file.setType(FTPFile.DIRECTORY_TYPE); 269 } else { 270 return null; 271 } 272 273 return file; 274 } 275 276 final boolean migrated = entry.startsWith("Migrated"); 277 if (migrated || entry.startsWith("ARCIVE")) { 278 // Type of file is unknown for migrated datasets 279 final FTPFile file = new FTPFile(); 280 file.setRawListing(entry); 281 file.setType(FTPFile.UNKNOWN_TYPE); 282 file.setName(entry.split("\\s+")[migrated ? 1 : 5]); 283 return file; 284 } 285 286 return null; 287 } 288 289 /** 290 * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code>FTPFile</code> instance. If the 291 * file listing line doesn't describe a file, then <code>null</code> is returned. Otherwise, a <code>FTPFile</code> instance representing the file is 292 * returned. 293 * 294 * @param entry A line of text from the file listing 295 * @return An FTPFile instance corresponding to the supplied entry 296 */ 297 @Override 298 public FTPFile parseFTPEntry(final String entry) { 299 switch (isType) { 300 case FILE_LIST_TYPE: 301 return parseFileList(entry); 302 case MEMBER_LIST_TYPE: 303 return parseMemberList(entry); 304 case UNIX_LIST_TYPE: 305 return unixFTPEntryParser.parseFTPEntry(entry); 306 case JES_LEVEL_1_LIST_TYPE: 307 return parseJeslevel1List(entry); 308 case JES_LEVEL_2_LIST_TYPE: 309 return parseJeslevel2List(entry); 310 default: 311 break; 312 } 313 314 return null; 315 } 316 317 /** 318 * Matches these entries, note: no header: 319 * 320 * <pre> 321 * [1] [2] [3] [4] [5] 322 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 323 * 012345678901234567890123456789012345678901234 324 * 1 2 3 4 325 * ------------------------------------------- 326 * Group in regex 327 * [1] Job name 328 * [2] Job number 329 * [3] Job status (INPUT,ACTIVE,OUTPUT) 330 * [4] Number of sysout files 331 * [5] The string "Spool Files" 332 * </pre> 333 * 334 * @param entry zosDirectoryEntry 335 * @return null: entry was not parsed. 336 */ 337 private FTPFile parseJeslevel1List(final String entry) { 338 return parseJeslevelList(entry, 3); 339 } 340 341 /** 342 * Matches these entries: 343 * 344 * <pre> 345 * [1] [2] [3] [4] [5] 346 * JOBNAME JOBID OWNER STATUS CLASS 347 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 348 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 349 * 012345678901234567890123456789012345678901234 350 * 1 2 3 4 351 * ------------------------------------------- 352 * Group in regex 353 * [1] Job name 354 * [2] Job number 355 * [3] Owner 356 * [4] Job status (INPUT,ACTIVE,OUTPUT) 357 * [5] Job Class 358 * [6] The rest 359 * </pre> 360 * 361 * @param entry zosDirectoryEntry 362 * @return null: entry was not parsed. 363 */ 364 private FTPFile parseJeslevel2List(final String entry) { 365 return parseJeslevelList(entry, 4); 366 } 367 368 private FTPFile parseJeslevelList(final String entry, final int matchNum) { 369 if (matches(entry)) { 370 final FTPFile file = new FTPFile(); 371 if (group(matchNum).equalsIgnoreCase("OUTPUT")) { 372 file.setRawListing(entry); 373 final String name = group(2); /* Job Number, used by GET */ 374 file.setName(name); 375 file.setType(FTPFile.FILE_TYPE); 376 return file; 377 } 378 } 379 return null; 380 } 381 382 /** 383 * Parses entries within a partitioned dataset. 384 * 385 * Format of a memberlist within a PDS: 386 * 387 * <pre> 388 * 0 1 2 3 4 5 6 7 8 389 * Name VV.MM Created Changed Size Init Mod Id 390 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 391 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 392 * 393 * ------------------------------------------- 394 * [1] Name 395 * [2] VV.MM: Version . modification 396 * [3] Created: yyyy / MM / dd 397 * [4,5] Changed: yyyy / MM / dd HH:mm 398 * [6] Size: number of lines 399 * [7] Init: number of lines when first created 400 * [8] Mod: number of modified lines a last save 401 * [9] Id: User id for last update 402 * </pre> 403 * 404 * @param entry zosDirectoryEntry 405 * @return null: entry was not parsed. 406 */ 407 private FTPFile parseMemberList(final String entry) { 408 final FTPFile file = new FTPFile(); 409 if (matches(entry)) { 410 file.setRawListing(entry); 411 final String name = group(1); 412 final String datestr = group(2) + " " + group(3); 413 file.setName(name); 414 file.setType(FTPFile.FILE_TYPE); 415 try { 416 file.setTimestamp(super.parseTimestamp(datestr)); 417 } catch (final ParseException e) { 418 // just ignore parsing errors. 419 // TODO check this is ok 420 // Drop thru to try simple parser 421 } 422 return file; 423 } 424 425 /* 426 * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason 427 * fails. Then just assign the name field of FTPFile. 428 */ 429 if (entry != null && !entry.trim().isEmpty()) { 430 file.setRawListing(entry); 431 final String name = entry.split(" ")[0]; 432 file.setName(name); 433 file.setType(FTPFile.FILE_TYPE); 434 return file; 435 } 436 return null; 437 } 438 439 /** 440 * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized: 441 * <ul> 442 * <li>z/OS-MVS File lists,</li> 443 * <li>z/OS-MVS Member lists,</li> 444 * <li>unix file lists.</li> 445 * </ul> 446 * @since 2.0 447 */ 448 @Override 449 public List<String> preParse(final List<String> orig) { 450 // simply remove the header line. Composite logic will take care of the 451 // two different types of 452 // list in short order. 453 if (orig != null && !orig.isEmpty()) { 454 final String header = orig.get(0); 455 if (header.contains("Volume") && header.contains("Dsname")) { 456 setType(FILE_LIST_TYPE); 457 super.setRegex(FILE_LIST_REGEX); 458 } else if (header.contains("Name") && header.contains("Id")) { 459 setType(MEMBER_LIST_TYPE); 460 super.setRegex(MEMBER_LIST_REGEX); 461 } else if (header.startsWith("total")) { 462 setType(UNIX_LIST_TYPE); 463 unixFTPEntryParser = new UnixFTPEntryParser(); 464 } else if (header.indexOf("Spool Files") >= 30) { 465 setType(JES_LEVEL_1_LIST_TYPE); 466 super.setRegex(JES_LEVEL_1_LIST_REGEX); 467 } else if (header.startsWith("JOBNAME") && header.indexOf("JOBID") > 8) { // header contains JOBNAME JOBID OWNER // STATUS CLASS 468 setType(JES_LEVEL_2_LIST_TYPE); 469 super.setRegex(JES_LEVEL_2_LIST_REGEX); 470 } else { 471 setType(UNKNOWN_LIST_TYPE); 472 } 473 474 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 475 orig.remove(0); 476 } 477 } 478 479 return orig; 480 } 481 482 /** 483 * Sets the type of listing being processed. 484 * 485 * @param type The listing type. 486 */ 487 void setType(final int type) { 488 isType = type; 489 } 490 491}