1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18 package org.apache.commons.net.ftp.parser;
19
20 import java.text.ParseException;
21 import java.util.List;
22
23 import org.apache.commons.net.ftp.Configurable;
24 import org.apache.commons.net.ftp.FTPClientConfig;
25 import org.apache.commons.net.ftp.FTPFile;
26 import org.apache.commons.net.ftp.FTPFileEntryParser;
27
28 /**
29 * Implements {@link FTPFileEntryParser} and {@link Configurable} for IBM zOS/MVS Systems.
30 *
31 * @see FTPFileEntryParser Usage instructions.
32 */
33 public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
34
35 static final int UNKNOWN_LIST_TYPE = -1;
36 static final int FILE_LIST_TYPE = 0;
37 static final int MEMBER_LIST_TYPE = 1;
38 static final int UNIX_LIST_TYPE = 2;
39 static final int JES_LEVEL_1_LIST_TYPE = 3;
40 static final int JES_LEVEL_2_LIST_TYPE = 4;
41
42 /**
43 * Dates are ignored for file lists, but are used for member lists where possible
44 */
45 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
46 // 13:52
47
48 /**
49 * Matches these entries:
50 *
51 * <pre>
52 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
53 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
54 * </pre>
55 *
56 * @see <a href= "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data set record formats</a>
57 */
58 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
59 // ignored
60 "\\S+\\s+" + // unit - ignored
61 "\\S+\\s+" + // access date - ignored
62 "\\S+\\s+" + // extents -ignored
63 // If the values are too large, the fields may be merged (NET-639)
64 "(?:\\S+\\s+)?" + // used - ignored
65 "\\S+\\s+" + // recfm - ignored
66 "\\S+\\s+" + // logical record length -ignored
67 "\\S+\\s+" + // block size - ignored
68 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
69 // but only support: PS, PO, PO-E
70 "(\\S+)\\s*"; // Dataset Name (file name)
71
72 /**
73 * Matches these entries:
74 *
75 * <pre>
76 * Name VV.MM Created Changed Size Init Mod Id
77 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
78 * </pre>
79 */
80 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
81 "\\S+\\s+" + // version, modification (ignored)
82 "\\S+\\s+" + // create date (ignored)
83 "(\\S+)\\s+" + // modification date
84 "(\\S+)\\s+" + // modification time
85 "\\S+\\s+" + // size in lines (ignored)
86 "\\S+\\s+" + // size in lines at creation(ignored)
87 "\\S+\\s+" + // lines modified (ignored)
88 "\\S+\\s*"; // id of user who modified (ignored)
89
90 /**
91 * Matches these entries, note: no header:
92 *
93 * <pre>
94 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
95 * 012345678901234567890123456789012345678901234
96 * 1 2 3 4
97 * </pre>
98 */
99 static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
100 "(\\S+)\\s+" + // job number
101 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
102 "(\\S+)\\s+" + // number of spool files
103 "(\\S+)\\s+" + // Text "Spool" ignored
104 "(\\S+)\\s*" // Text "Files" ignored
105 ;
106
107 /**
108 * JES INTERFACE LEVEL 2 parser Matches these entries:
109 *
110 * <pre>
111 * JOBNAME JOBID OWNER STATUS CLASS
112 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
113 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
114 * </pre>
115 *
116 * Sample output from FTP session:
117 *
118 * <pre>
119 * ftp> quote site filetype=jes
120 * 200 SITE command was accepted
121 * ftp> ls
122 * 200 Port request OK.
123 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
124 * JOBNAME JOBID OWNER STATUS CLASS
125 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
126 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
127 * 250 List completed successfully.
128 * ftp> ls job01906
129 * 200 Port request OK.
130 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
131 * JOBNAME JOBID OWNER STATUS CLASS
132 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000
133 * --------
134 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT
135 * 001 JES2 A JESMSGLG 858
136 * 002 JES2 A JESJCL 128
137 * 003 JES2 A JESYSMSG 443
138 * 3 spool files
139 * 250 List completed successfully.
140 * </pre>
141 */
142
143 static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job name ignored
144 "(\\S+)\\s+" + // job number
145 "(\\S+)\\s+" + // owner ignored
146 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
147 "(\\S+)\\s+" + // job class ignored
148 "(\\S+).*" // rest ignored
149 ;
150
151 private int isType = UNKNOWN_LIST_TYPE;
152
153 /**
154 * Fallback parser for Unix-style listings
155 */
156 private UnixFTPEntryParser unixFTPEntryParser;
157
158 /*
159 * --------------------------------------------------------------------- Very brief and incomplete description of the zOS/MVS-file system. (Note: "zOS" is
160 * the operating system on the mainframe, and is the new name for MVS)
161 *
162 * The file system on the mainframe does not have hierarchical structure as for example the unix file system. For a more comprehensive description,
163 * please refer to the IBM manuals
164 *
165 * @LINK: https://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
166 *
167 *
168 * Dataset names =============
169 *
170 * A dataset name consist of a number of qualifiers separated by '.', each qualifier can be at most 8 characters, and the total length of a dataset can be
171 * max 44 characters including the dots.
172 *
173 *
174 * Dataset organisation ====================
175 *
176 * A dataset represents a piece of storage allocated on one or more disks. The structure of the storage is described with the field dataset organisation
177 * (DSORG). There are a number of dataset organisations, but only two are usable for FTP transfer.
178 *
179 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: extended partitioned dataset
180 *
181 * The PS file is just a flat file, as you would find it on the unix file system.
182 *
183 * The PO and PO-E files, can be compared to a single level directory structure. A PO file consist of a number of dataset members, or files if you will. It
184 * is possible to CD into the file, and to retrieve the individual members.
185 *
186 *
187 * Dataset record format =====================
188 *
189 * The physical layout of the dataset is described on the dataset itself. There are a number of record formats (RECFM), but just a few is relevant for the
190 * FTP transfer.
191 *
192 * Any one beginning with either F or V can safely be used by FTP transfer. All others should only be used with great care. F means a fixed number of
193 * records per allocated storage, and V means a variable number of records.
194 *
195 *
196 * Other notes ===========
197 *
198 * The file system supports automatically backup and retrieval of datasets. If a file is backed up, the ftp LIST command will return: ARCIVE Not Direct
199 * Access Device KJ.IOP998.ERROR.PL.UNITTEST
200 *
201 *
202 * Implementation notes ====================
203 *
204 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning with F or V or U, is fully parsed.
205 *
206 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
207 * FTPFile.Timestamp: change time or null
208 *
209 *
210 *
211 * Additional information ======================
212 *
213 * The MVS ftp server supports a number of features via the FTP interface. The features are controlled with the FTP command quote site
214 * filetype=<SEQ|JES|DB2> SEQ is the default and used for normal file transfer JES is used to interact with the Job Entry Subsystem (JES) similar to a job
215 * scheduler DB2 is used to interact with a DB2 subsystem
216 *
217 * This parser supports SEQ and JES.
218 */
219
220 /**
221 * The sole constructor for a MVSFTPEntryParser object.
222 */
223 public MVSFTPEntryParser() {
224 super(""); // note the regex is set in preParse.
225 super.configure(null); // configure parser with default configurations
226 }
227
228 /*
229 * @return
230 */
231 @Override
232 protected FTPClientConfig getDefaultConfiguration() {
233 return new FTPClientConfig(FTPClientConfig.SYST_MVS, DEFAULT_DATE_FORMAT, null);
234 }
235
236 /**
237 * Parses entries representing a dataset list.
238 * <pre>
239 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10
240 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
241 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
242 * ARCIVE Not Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
243 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO PLU
244 * B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
245 * Migrated HLQ.DATASET.NAME
246 * </pre>
247 * <pre>
248 * ----------------------------------- Group within Regex [1] Volume [2] Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record format [7]
249 * Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: Dataset organisation. Many exists but only support: PS, PO, PO-E [10] Dsname: Dataset name
250 * </pre>
251 *
252 * @param entry zosDirectoryEntry
253 * @return null: entry was not parsed.
254 */
255 private FTPFile parseFileList(final String entry) {
256 if (matches(entry)) {
257 final FTPFile file = new FTPFile();
258 file.setRawListing(entry);
259 final String name = group(2);
260 final String dsorg = group(1);
261 file.setName(name);
262
263 // DSORG
264 if ("PS".equals(dsorg)) {
265 file.setType(FTPFile.FILE_TYPE);
266 } else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
267 // regex already ruled out anything other than PO or PO-E
268 file.setType(FTPFile.DIRECTORY_TYPE);
269 } else {
270 return null;
271 }
272
273 return file;
274 }
275
276 final boolean migrated = entry.startsWith("Migrated");
277 if (migrated || entry.startsWith("ARCIVE")) {
278 // Type of file is unknown for migrated datasets
279 final FTPFile file = new FTPFile();
280 file.setRawListing(entry);
281 file.setType(FTPFile.UNKNOWN_TYPE);
282 file.setName(entry.split("\\s+")[migrated ? 1 : 5]);
283 return file;
284 }
285
286 return null;
287 }
288
289 /**
290 * Parses a line of a z/OS - MVS FTP server file listing and converts it into a usable format in the form of an <code>FTPFile</code> instance. If the
291 * file listing line doesn't describe a file, then <code>null</code> is returned. Otherwise, a <code>FTPFile</code> instance representing the file is
292 * returned.
293 *
294 * @param entry A line of text from the file listing
295 * @return An FTPFile instance corresponding to the supplied entry
296 */
297 @Override
298 public FTPFile parseFTPEntry(final String entry) {
299 switch (isType) {
300 case FILE_LIST_TYPE:
301 return parseFileList(entry);
302 case MEMBER_LIST_TYPE:
303 return parseMemberList(entry);
304 case UNIX_LIST_TYPE:
305 return unixFTPEntryParser.parseFTPEntry(entry);
306 case JES_LEVEL_1_LIST_TYPE:
307 return parseJeslevel1List(entry);
308 case JES_LEVEL_2_LIST_TYPE:
309 return parseJeslevel2List(entry);
310 default:
311 break;
312 }
313
314 return null;
315 }
316
317 /**
318 * Matches these entries, note: no header:
319 *
320 * <pre>
321 * [1] [2] [3] [4] [5]
322 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
323 * 012345678901234567890123456789012345678901234
324 * 1 2 3 4
325 * -------------------------------------------
326 * Group in regex
327 * [1] Job name
328 * [2] Job number
329 * [3] Job status (INPUT,ACTIVE,OUTPUT)
330 * [4] Number of sysout files
331 * [5] The string "Spool Files"
332 * </pre>
333 *
334 * @param entry zosDirectoryEntry
335 * @return null: entry was not parsed.
336 */
337 private FTPFile parseJeslevel1List(final String entry) {
338 return parseJeslevelList(entry, 3);
339 }
340
341 /**
342 * Matches these entries:
343 *
344 * <pre>
345 * [1] [2] [3] [4] [5]
346 * JOBNAME JOBID OWNER STATUS CLASS
347 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
348 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
349 * 012345678901234567890123456789012345678901234
350 * 1 2 3 4
351 * -------------------------------------------
352 * Group in regex
353 * [1] Job name
354 * [2] Job number
355 * [3] Owner
356 * [4] Job status (INPUT,ACTIVE,OUTPUT)
357 * [5] Job Class
358 * [6] The rest
359 * </pre>
360 *
361 * @param entry zosDirectoryEntry
362 * @return null: entry was not parsed.
363 */
364 private FTPFile parseJeslevel2List(final String entry) {
365 return parseJeslevelList(entry, 4);
366 }
367
368 private FTPFile parseJeslevelList(final String entry, final int matchNum) {
369 if (matches(entry)) {
370 final FTPFile file = new FTPFile();
371 if (group(matchNum).equalsIgnoreCase("OUTPUT")) {
372 file.setRawListing(entry);
373 final String name = group(2); /* Job Number, used by GET */
374 file.setName(name);
375 file.setType(FTPFile.FILE_TYPE);
376 return file;
377 }
378 }
379 return null;
380 }
381
382 /**
383 * Parses entries within a partitioned dataset.
384 *
385 * Format of a memberlist within a PDS:
386 *
387 * <pre>
388 * 0 1 2 3 4 5 6 7 8
389 * Name VV.MM Created Changed Size Init Mod Id
390 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
391 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001
392 *
393 * -------------------------------------------
394 * [1] Name
395 * [2] VV.MM: Version . modification
396 * [3] Created: yyyy / MM / dd
397 * [4,5] Changed: yyyy / MM / dd HH:mm
398 * [6] Size: number of lines
399 * [7] Init: number of lines when first created
400 * [8] Mod: number of modified lines a last save
401 * [9] Id: User id for last update
402 * </pre>
403 *
404 * @param entry zosDirectoryEntry
405 * @return null: entry was not parsed.
406 */
407 private FTPFile parseMemberList(final String entry) {
408 final FTPFile file = new FTPFile();
409 if (matches(entry)) {
410 file.setRawListing(entry);
411 final String name = group(1);
412 final String datestr = group(2) + " " + group(3);
413 file.setName(name);
414 file.setType(FTPFile.FILE_TYPE);
415 try {
416 file.setTimestamp(super.parseTimestamp(datestr));
417 } catch (final ParseException e) {
418 // just ignore parsing errors.
419 // TODO check this is ok
420 // Drop thru to try simple parser
421 }
422 return file;
423 }
424
425 /*
426 * Assigns the name to the first word of the entry. Only to be used from a safe context, for example from a memberlist, where the regex for some reason
427 * fails. Then just assign the name field of FTPFile.
428 */
429 if (entry != null && !entry.trim().isEmpty()) {
430 file.setRawListing(entry);
431 final String name = entry.split(" ")[0];
432 file.setName(name);
433 file.setType(FTPFile.FILE_TYPE);
434 return file;
435 }
436 return null;
437 }
438
439 /**
440 * Pre-parses is called as part of the interface. Per definition, it is called before the parsing takes place. Three kinds of lists are recognized:
441 * <ul>
442 * <li>z/OS-MVS File lists,</li>
443 * <li>z/OS-MVS Member lists,</li>
444 * <li>unix file lists.</li>
445 * </ul>
446 * @since 2.0
447 */
448 @Override
449 public List<String> preParse(final List<String> orig) {
450 // simply remove the header line. Composite logic will take care of the
451 // two different types of
452 // list in short order.
453 if (orig != null && !orig.isEmpty()) {
454 final String header = orig.get(0);
455 if (header.contains("Volume") && header.contains("Dsname")) {
456 setType(FILE_LIST_TYPE);
457 super.setRegex(FILE_LIST_REGEX);
458 } else if (header.contains("Name") && header.contains("Id")) {
459 setType(MEMBER_LIST_TYPE);
460 super.setRegex(MEMBER_LIST_REGEX);
461 } else if (header.startsWith("total")) {
462 setType(UNIX_LIST_TYPE);
463 unixFTPEntryParser = new UnixFTPEntryParser();
464 } else if (header.indexOf("Spool Files") >= 30) {
465 setType(JES_LEVEL_1_LIST_TYPE);
466 super.setRegex(JES_LEVEL_1_LIST_REGEX);
467 } else if (header.startsWith("JOBNAME") && header.indexOf("JOBID") > 8) { // header contains JOBNAME JOBID OWNER // STATUS CLASS
468 setType(JES_LEVEL_2_LIST_TYPE);
469 super.setRegex(JES_LEVEL_2_LIST_REGEX);
470 } else {
471 setType(UNKNOWN_LIST_TYPE);
472 }
473
474 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
475 orig.remove(0);
476 }
477 }
478
479 return orig;
480 }
481
482 /**
483 * Sets the type of listing being processed.
484 *
485 * @param type The listing type.
486 */
487 void setType(final int type) {
488 isType = type;
489 }
490
491 }