001 /*
002 * Copyright 2008-2017 UnboundID Corp.
003 * All Rights Reserved.
004 */
005 /*
006 * Copyright (C) 2008-2017 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021 package com.unboundid.ldap.sdk.examples;
022
023
024
025 import java.io.IOException;
026 import java.io.OutputStream;
027 import java.io.Serializable;
028 import java.text.ParseException;
029 import java.util.ArrayList;
030 import java.util.LinkedHashMap;
031 import java.util.LinkedHashSet;
032 import java.util.List;
033 import java.util.StringTokenizer;
034 import java.util.concurrent.CyclicBarrier;
035 import java.util.concurrent.Semaphore;
036 import java.util.concurrent.atomic.AtomicBoolean;
037 import java.util.concurrent.atomic.AtomicLong;
038
039 import com.unboundid.ldap.sdk.Control;
040 import com.unboundid.ldap.sdk.LDAPConnection;
041 import com.unboundid.ldap.sdk.LDAPConnectionOptions;
042 import com.unboundid.ldap.sdk.LDAPException;
043 import com.unboundid.ldap.sdk.ResultCode;
044 import com.unboundid.ldap.sdk.SearchScope;
045 import com.unboundid.ldap.sdk.Version;
046 import com.unboundid.ldap.sdk.controls.AssertionRequestControl;
047 import com.unboundid.ldap.sdk.controls.ServerSideSortRequestControl;
048 import com.unboundid.ldap.sdk.controls.SortKey;
049 import com.unboundid.util.ColumnFormatter;
050 import com.unboundid.util.FixedRateBarrier;
051 import com.unboundid.util.FormattableColumn;
052 import com.unboundid.util.HorizontalAlignment;
053 import com.unboundid.util.LDAPCommandLineTool;
054 import com.unboundid.util.ObjectPair;
055 import com.unboundid.util.OutputFormat;
056 import com.unboundid.util.RateAdjustor;
057 import com.unboundid.util.ResultCodeCounter;
058 import com.unboundid.util.ThreadSafety;
059 import com.unboundid.util.ThreadSafetyLevel;
060 import com.unboundid.util.WakeableSleeper;
061 import com.unboundid.util.ValuePattern;
062 import com.unboundid.util.args.ArgumentException;
063 import com.unboundid.util.args.ArgumentParser;
064 import com.unboundid.util.args.BooleanArgument;
065 import com.unboundid.util.args.ControlArgument;
066 import com.unboundid.util.args.FileArgument;
067 import com.unboundid.util.args.FilterArgument;
068 import com.unboundid.util.args.IntegerArgument;
069 import com.unboundid.util.args.ScopeArgument;
070 import com.unboundid.util.args.StringArgument;
071
072 import static com.unboundid.util.Debug.*;
073 import static com.unboundid.util.StaticUtils.*;
074
075
076
077 /**
078 * This class provides a tool that can be used to search an LDAP directory
079 * server repeatedly using multiple threads. It can help provide an estimate of
080 * the search performance that a directory server is able to achieve. Either or
081 * both of the base DN and the search filter may be a value pattern as
082 * described in the {@link ValuePattern} class. This makes it possible to
083 * search over a range of entries rather than repeatedly performing searches
084 * with the same base DN and filter.
085 * <BR><BR>
086 * Some of the APIs demonstrated by this example include:
087 * <UL>
088 * <LI>Argument Parsing (from the {@code com.unboundid.util.args}
089 * package)</LI>
090 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
091 * package)</LI>
092 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
093 * package)</LI>
094 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
095 * </UL>
096 * <BR><BR>
097 * All of the necessary information is provided using command line arguments.
098 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
099 * class, as well as the following additional arguments:
100 * <UL>
101 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
102 * for the searches. This must be provided. It may be a simple DN, or it
103 * may be a value pattern to express a range of base DNs.</LI>
104 * <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
105 * search. The scope value should be one of "base", "one", "sub", or
106 * "subord". If this isn't specified, then a scope of "sub" will be
107 * used.</LI>
108 * <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
109 * the searches. This must be provided. It may be a simple filter, or it
110 * may be a value pattern to express a range of filters.</LI>
111 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
112 * attribute that should be included in entries returned from the server.
113 * If this is not provided, then all user attributes will be requested.
114 * This may include special tokens that the server may interpret, like
115 * "1.1" to indicate that no attributes should be returned, "*", for all
116 * user attributes, or "+" for all operational attributes. Multiple
117 * attributes may be requested with multiple instances of this
118 * argument.</LI>
119 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
120 * concurrent threads to use when performing the searches. If this is not
121 * provided, then a default of one thread will be used.</LI>
122 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
123 * time in seconds between lines out output. If this is not provided,
124 * then a default interval duration of five seconds will be used.</LI>
125 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
126 * intervals for which to run. If this is not provided, then it will
127 * run forever.</LI>
128 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
129 * iterations that should be performed on a connection before that
130 * connection is closed and replaced with a newly-established (and
131 * authenticated, if appropriate) connection.</LI>
132 * <LI>"-r {searches-per-second}" or "--ratePerSecond {searches-per-second}"
133 * -- specifies the target number of searches to perform per second. It
134 * is still necessary to specify a sufficient number of threads for
135 * achieving this rate. If this option is not provided, then the tool
136 * will run at the maximum rate for the specified number of threads.</LI>
137 * <LI>"--variableRateData {path}" -- specifies the path to a file containing
138 * information needed to allow the tool to vary the target rate over time.
139 * If this option is not provided, then the tool will either use a fixed
140 * target rate as specified by the "--ratePerSecond" argument, or it will
141 * run at the maximum rate.</LI>
142 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
143 * which sample data will be written illustrating and describing the
144 * format of the file expected to be used in conjunction with the
145 * "--variableRateData" argument.</LI>
146 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
147 * complete before beginning overall statistics collection.</LI>
148 * <LI>"--timestampFormat {format}" -- specifies the format to use for
149 * timestamps included before each output line. The format may be one of
150 * "none" (for no timestamps), "with-date" (to include both the date and
151 * the time), or "without-date" (to include only time time).</LI>
152 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
153 * authorization v2 control to request that the operation be processed
154 * using an alternate authorization identity. In this case, the bind DN
155 * should be that of a user that has permission to use this control. The
156 * authorization identity may be a value pattern.</LI>
157 * <LI>"-a" or "--asynchronous" -- Indicates that searches should be performed
158 * in asynchronous mode, in which the client will not wait for a response
159 * to a previous request before sending the next request. Either the
160 * "--ratePerSecond" or "--maxOutstandingRequests" arguments must be
161 * provided to limit the number of outstanding requests.</LI>
162 * <LI>"-O {num}" or "--maxOutstandingRequests {num}" -- Specifies the maximum
163 * number of outstanding requests that will be allowed in asynchronous
164 * mode.</LI>
165 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the
166 * result codes for failed operations should not be displayed.</LI>
167 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
168 * display-friendly format.</LI>
169 * </UL>
170 */
171 @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
172 public final class SearchRate
173 extends LDAPCommandLineTool
174 implements Serializable
175 {
176 /**
177 * The serial version UID for this serializable class.
178 */
179 private static final long serialVersionUID = 3345838530404592182L;
180
181
182
183 // Indicates whether a request has been made to stop running.
184 private final AtomicBoolean stopRequested;
185
186 // The argument used to indicate whether to operate in asynchronous mode.
187 private BooleanArgument asynchronousMode;
188
189 // The argument used to indicate whether to generate output in CSV format.
190 private BooleanArgument csvFormat;
191
192 // The argument used to indicate whether to suppress information about error
193 // result codes.
194 private BooleanArgument suppressErrors;
195
196 // The argument used to indicate that a generic control should be included in
197 // the request.
198 private ControlArgument control;
199
200 // The argument used to specify a variable rate file.
201 private FileArgument sampleRateFile;
202
203 // The argument used to specify a variable rate file.
204 private FileArgument variableRateData;
205
206 // Indicates that search requests should include the assertion request control
207 // with the specified filter.
208 private FilterArgument assertionFilter;
209
210 // The argument used to specify the collection interval.
211 private IntegerArgument collectionInterval;
212
213 // The argument used to specify the number of search iterations on a
214 // connection before it is closed and re-established.
215 private IntegerArgument iterationsBeforeReconnect;
216
217 // The argument used to specify the maximum number of outstanding asynchronous
218 // requests.
219 private IntegerArgument maxOutstandingRequests;
220
221 // The argument used to specify the number of intervals.
222 private IntegerArgument numIntervals;
223
224 // The argument used to specify the number of threads.
225 private IntegerArgument numThreads;
226
227 // The argument used to specify the seed to use for the random number
228 // generator.
229 private IntegerArgument randomSeed;
230
231 // The target rate of searches per second.
232 private IntegerArgument ratePerSecond;
233
234 // The argument used to indicate that the search should use the simple paged
235 // results control with the specified page size.
236 private IntegerArgument simplePageSize;
237
238 // The number of warm-up intervals to perform.
239 private IntegerArgument warmUpIntervals;
240
241 // The argument used to specify the scope for the searches.
242 private ScopeArgument scopeArg;
243
244 // The argument used to specify the attributes to return.
245 private StringArgument attributes;
246
247 // The argument used to specify the base DNs for the searches.
248 private StringArgument baseDN;
249
250 // The argument used to specify the filters for the searches.
251 private StringArgument filter;
252
253 // The argument used to specify the proxied authorization identity.
254 private StringArgument proxyAs;
255
256 // The argument used to request that the server sort the results with the
257 // specified order.
258 private StringArgument sortOrder;
259
260 // The argument used to specify the timestamp format.
261 private StringArgument timestampFormat;
262
263 // The thread currently being used to run the searchrate tool.
264 private volatile Thread runningThread;
265
266 // A wakeable sleeper that will be used to sleep between reporting intervals.
267 private final WakeableSleeper sleeper;
268
269
270
271 /**
272 * Parse the provided command line arguments and make the appropriate set of
273 * changes.
274 *
275 * @param args The command line arguments provided to this program.
276 */
277 public static void main(final String[] args)
278 {
279 final ResultCode resultCode = main(args, System.out, System.err);
280 if (resultCode != ResultCode.SUCCESS)
281 {
282 System.exit(resultCode.intValue());
283 }
284 }
285
286
287
288 /**
289 * Parse the provided command line arguments and make the appropriate set of
290 * changes.
291 *
292 * @param args The command line arguments provided to this program.
293 * @param outStream The output stream to which standard out should be
294 * written. It may be {@code null} if output should be
295 * suppressed.
296 * @param errStream The output stream to which standard error should be
297 * written. It may be {@code null} if error messages
298 * should be suppressed.
299 *
300 * @return A result code indicating whether the processing was successful.
301 */
302 public static ResultCode main(final String[] args,
303 final OutputStream outStream,
304 final OutputStream errStream)
305 {
306 final SearchRate searchRate = new SearchRate(outStream, errStream);
307 return searchRate.runTool(args);
308 }
309
310
311
312 /**
313 * Creates a new instance of this tool.
314 *
315 * @param outStream The output stream to which standard out should be
316 * written. It may be {@code null} if output should be
317 * suppressed.
318 * @param errStream The output stream to which standard error should be
319 * written. It may be {@code null} if error messages
320 * should be suppressed.
321 */
322 public SearchRate(final OutputStream outStream, final OutputStream errStream)
323 {
324 super(outStream, errStream);
325
326 stopRequested = new AtomicBoolean(false);
327 sleeper = new WakeableSleeper();
328 }
329
330
331
332 /**
333 * Retrieves the name for this tool.
334 *
335 * @return The name for this tool.
336 */
337 @Override()
338 public String getToolName()
339 {
340 return "searchrate";
341 }
342
343
344
345 /**
346 * Retrieves the description for this tool.
347 *
348 * @return The description for this tool.
349 */
350 @Override()
351 public String getToolDescription()
352 {
353 return "Perform repeated searches against an " +
354 "LDAP directory server.";
355 }
356
357
358
359 /**
360 * Retrieves the version string for this tool.
361 *
362 * @return The version string for this tool.
363 */
364 @Override()
365 public String getToolVersion()
366 {
367 return Version.NUMERIC_VERSION_STRING;
368 }
369
370
371
372 /**
373 * Indicates whether this tool should provide support for an interactive mode,
374 * in which the tool offers a mode in which the arguments can be provided in
375 * a text-driven menu rather than requiring them to be given on the command
376 * line. If interactive mode is supported, it may be invoked using the
377 * "--interactive" argument. Alternately, if interactive mode is supported
378 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
379 * interactive mode may be invoked by simply launching the tool without any
380 * arguments.
381 *
382 * @return {@code true} if this tool supports interactive mode, or
383 * {@code false} if not.
384 */
385 @Override()
386 public boolean supportsInteractiveMode()
387 {
388 return true;
389 }
390
391
392
393 /**
394 * Indicates whether this tool defaults to launching in interactive mode if
395 * the tool is invoked without any command-line arguments. This will only be
396 * used if {@link #supportsInteractiveMode()} returns {@code true}.
397 *
398 * @return {@code true} if this tool defaults to using interactive mode if
399 * launched without any command-line arguments, or {@code false} if
400 * not.
401 */
402 @Override()
403 public boolean defaultsToInteractiveMode()
404 {
405 return true;
406 }
407
408
409
410 /**
411 * Indicates whether this tool should provide arguments for redirecting output
412 * to a file. If this method returns {@code true}, then the tool will offer
413 * an "--outputFile" argument that will specify the path to a file to which
414 * all standard output and standard error content will be written, and it will
415 * also offer a "--teeToStandardOut" argument that can only be used if the
416 * "--outputFile" argument is present and will cause all output to be written
417 * to both the specified output file and to standard output.
418 *
419 * @return {@code true} if this tool should provide arguments for redirecting
420 * output to a file, or {@code false} if not.
421 */
422 @Override()
423 protected boolean supportsOutputFile()
424 {
425 return true;
426 }
427
428
429
430 /**
431 * Indicates whether this tool should default to interactively prompting for
432 * the bind password if a password is required but no argument was provided
433 * to indicate how to get the password.
434 *
435 * @return {@code true} if this tool should default to interactively
436 * prompting for the bind password, or {@code false} if not.
437 */
438 @Override()
439 protected boolean defaultToPromptForBindPassword()
440 {
441 return true;
442 }
443
444
445
446 /**
447 * Indicates whether this tool supports the use of a properties file for
448 * specifying default values for arguments that aren't specified on the
449 * command line.
450 *
451 * @return {@code true} if this tool supports the use of a properties file
452 * for specifying default values for arguments that aren't specified
453 * on the command line, or {@code false} if not.
454 */
455 @Override()
456 public boolean supportsPropertiesFile()
457 {
458 return true;
459 }
460
461
462
463 /**
464 * Indicates whether the LDAP-specific arguments should include alternate
465 * versions of all long identifiers that consist of multiple words so that
466 * they are available in both camelCase and dash-separated versions.
467 *
468 * @return {@code true} if this tool should provide multiple versions of
469 * long identifiers for LDAP-specific arguments, or {@code false} if
470 * not.
471 */
472 @Override()
473 protected boolean includeAlternateLongIdentifiers()
474 {
475 return true;
476 }
477
478
479
480 /**
481 * Adds the arguments used by this program that aren't already provided by the
482 * generic {@code LDAPCommandLineTool} framework.
483 *
484 * @param parser The argument parser to which the arguments should be added.
485 *
486 * @throws ArgumentException If a problem occurs while adding the arguments.
487 */
488 @Override()
489 public void addNonLDAPArguments(final ArgumentParser parser)
490 throws ArgumentException
491 {
492 String description = "The base DN to use for the searches. It may be a " +
493 "simple DN or a value pattern to specify a range of DNs (e.g., " +
494 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " +
495 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
496 "value pattern syntax. This must be provided.";
497 baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
498 baseDN.setArgumentGroupName("Search Arguments");
499 baseDN.addLongIdentifier("base-dn");
500 parser.addArgument(baseDN);
501
502
503 description = "The scope to use for the searches. It should be 'base', " +
504 "'one', 'sub', or 'subord'. If this is not provided, then " +
505 "a default scope of 'sub' will be used.";
506 scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
507 SearchScope.SUB);
508 scopeArg.setArgumentGroupName("Search Arguments");
509 parser.addArgument(scopeArg);
510
511
512 description = "The filter to use for the searches. It may be a simple " +
513 "filter or a value pattern to specify a range of filters " +
514 "(e.g., \"(uid=user.[1-1000])\"). See " +
515 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details " +
516 "about the value pattern syntax. This must be provided.";
517 filter = new StringArgument('f', "filter", true, 1, "{filter}",
518 description);
519 filter.setArgumentGroupName("Search Arguments");
520 parser.addArgument(filter);
521
522
523 description = "The name of an attribute to include in entries returned " +
524 "from the searches. Multiple attributes may be requested " +
525 "by providing this argument multiple times. If no request " +
526 "attributes are provided, then the entries returned will " +
527 "include all user attributes.";
528 attributes = new StringArgument('A', "attribute", false, 0, "{name}",
529 description);
530 attributes.setArgumentGroupName("Search Arguments");
531 parser.addArgument(attributes);
532
533
534 description = "Indicates that search requests should include the " +
535 "assertion request control with the specified filter.";
536 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1,
537 "{filter}", description);
538 assertionFilter.setArgumentGroupName("Request Control Arguments");
539 assertionFilter.addLongIdentifier("assertion-filter");
540 parser.addArgument(assertionFilter);
541
542
543 description = "Indicates that search requests should include the simple " +
544 "paged results control with the specified page size.";
545 simplePageSize = new IntegerArgument(null, "simplePageSize", false, 1,
546 "{size}", description, 1,
547 Integer.MAX_VALUE);
548 simplePageSize.setArgumentGroupName("Request Control Arguments");
549 simplePageSize.addLongIdentifier("simple-page-size");
550 parser.addArgument(simplePageSize);
551
552
553 description = "Indicates that search requests should include the " +
554 "server-side sort request control with the specified sort " +
555 "order. This should be a comma-delimited list in which " +
556 "each item is an attribute name, optionally preceded by a " +
557 "plus or minus sign (to indicate ascending or descending " +
558 "order; where ascending order is the default), and " +
559 "optionally followed by a colon and the name or OID of " +
560 "the desired ordering matching rule (if this is not " +
561 "provided, the the attribute type's default ordering " +
562 "rule will be used).";
563 sortOrder = new StringArgument(null, "sortOrder", false, 1, "{sortOrder}",
564 description);
565 sortOrder.setArgumentGroupName("Request Control Arguments");
566 sortOrder.addLongIdentifier("sort-order");
567 parser.addArgument(sortOrder);
568
569
570 description = "Indicates that the proxied authorization control (as " +
571 "defined in RFC 4370) should be used to request that " +
572 "operations be processed using an alternate authorization " +
573 "identity. This may be a simple authorization ID or it " +
574 "may be a value pattern to specify a range of " +
575 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL +
576 " for complete details about the value pattern syntax.";
577 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
578 description);
579 proxyAs.setArgumentGroupName("Request Control Arguments");
580 proxyAs.addLongIdentifier("proxy-as");
581 parser.addArgument(proxyAs);
582
583
584 description = "Indicates that search requests should include the " +
585 "specified request control. This may be provided multiple " +
586 "times to include multiple request controls.";
587 control = new ControlArgument('J', "control", false, 0, null, description);
588 control.setArgumentGroupName("Request Control Arguments");
589 parser.addArgument(control);
590
591
592 description = "The number of threads to use to perform the searches. If " +
593 "this is not provided, then a default of one thread will " +
594 "be used.";
595 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
596 description, 1, Integer.MAX_VALUE, 1);
597 numThreads.setArgumentGroupName("Rate Management Arguments");
598 numThreads.addLongIdentifier("num-threads");
599 parser.addArgument(numThreads);
600
601
602 description = "The length of time in seconds between output lines. If " +
603 "this is not provided, then a default interval of five " +
604 "seconds will be used.";
605 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
606 "{num}", description, 1,
607 Integer.MAX_VALUE, 5);
608 collectionInterval.setArgumentGroupName("Rate Management Arguments");
609 collectionInterval.addLongIdentifier("interval-duration");
610 parser.addArgument(collectionInterval);
611
612
613 description = "The maximum number of intervals for which to run. If " +
614 "this is not provided, then the tool will run until it is " +
615 "interrupted.";
616 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
617 description, 1, Integer.MAX_VALUE,
618 Integer.MAX_VALUE);
619 numIntervals.setArgumentGroupName("Rate Management Arguments");
620 numIntervals.addLongIdentifier("num-intervals");
621 parser.addArgument(numIntervals);
622
623 description = "The number of search iterations that should be processed " +
624 "on a connection before that connection is closed and " +
625 "replaced with a newly-established (and authenticated, if " +
626 "appropriate) connection. If this is not provided, then " +
627 "connections will not be periodically closed and " +
628 "re-established.";
629 iterationsBeforeReconnect = new IntegerArgument(null,
630 "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
631 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
632 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect");
633 parser.addArgument(iterationsBeforeReconnect);
634
635 description = "The target number of searches to perform per second. It " +
636 "is still necessary to specify a sufficient number of " +
637 "threads for achieving this rate. If neither this option " +
638 "nor --variableRateData is provided, then the tool will " +
639 "run at the maximum rate for the specified number of " +
640 "threads.";
641 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
642 "{searches-per-second}", description,
643 1, Integer.MAX_VALUE);
644 ratePerSecond.setArgumentGroupName("Rate Management Arguments");
645 ratePerSecond.addLongIdentifier("rate-per-second");
646 parser.addArgument(ratePerSecond);
647
648 final String variableRateDataArgName = "variableRateData";
649 final String generateSampleRateFileArgName = "generateSampleRateFile";
650 description = RateAdjustor.getVariableRateDataArgumentDescription(
651 generateSampleRateFileArgName);
652 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
653 "{path}", description, true, true, true,
654 false);
655 variableRateData.setArgumentGroupName("Rate Management Arguments");
656 variableRateData.addLongIdentifier("variable-rate-data");
657 parser.addArgument(variableRateData);
658
659 description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
660 variableRateDataArgName);
661 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
662 false, 1, "{path}", description, false,
663 true, true, false);
664 sampleRateFile.setArgumentGroupName("Rate Management Arguments");
665 sampleRateFile.addLongIdentifier("generate-sample-rate-file");
666 sampleRateFile.setUsageArgument(true);
667 parser.addArgument(sampleRateFile);
668 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
669
670 description = "The number of intervals to complete before beginning " +
671 "overall statistics collection. Specifying a nonzero " +
672 "number of warm-up intervals gives the client and server " +
673 "a chance to warm up without skewing performance results.";
674 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
675 "{num}", description, 0, Integer.MAX_VALUE, 0);
676 warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
677 warmUpIntervals.addLongIdentifier("warm-up-intervals");
678 parser.addArgument(warmUpIntervals);
679
680 description = "Indicates the format to use for timestamps included in " +
681 "the output. A value of 'none' indicates that no " +
682 "timestamps should be included. A value of 'with-date' " +
683 "indicates that both the date and the time should be " +
684 "included. A value of 'without-date' indicates that only " +
685 "the time should be included.";
686 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
687 allowedFormats.add("none");
688 allowedFormats.add("with-date");
689 allowedFormats.add("without-date");
690 timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
691 "{format}", description, allowedFormats, "none");
692 timestampFormat.addLongIdentifier("timestamp-format");
693 parser.addArgument(timestampFormat);
694
695 description = "Indicates that the client should operate in asynchronous " +
696 "mode, in which it will not be necessary to wait for a " +
697 "response to a previous request before sending the next " +
698 "request. Either the '--ratePerSecond' or the " +
699 "'--maxOutstandingRequests' argument must be provided to " +
700 "limit the number of outstanding requests.";
701 asynchronousMode = new BooleanArgument('a', "asynchronous", description);
702 parser.addArgument(asynchronousMode);
703
704 description = "Specifies the maximum number of outstanding requests " +
705 "that should be allowed when operating in asynchronous mode.";
706 maxOutstandingRequests = new IntegerArgument('O', "maxOutstandingRequests",
707 false, 1, "{num}", description, 1, Integer.MAX_VALUE, (Integer) null);
708 maxOutstandingRequests.addLongIdentifier("max-outstanding-requests");
709 parser.addArgument(maxOutstandingRequests);
710
711 description = "Indicates that information about the result codes for " +
712 "failed operations should not be displayed.";
713 suppressErrors = new BooleanArgument(null,
714 "suppressErrorResultCodes", 1, description);
715 suppressErrors.addLongIdentifier("suppress-error-result-codes");
716 parser.addArgument(suppressErrors);
717
718 description = "Generate output in CSV format rather than a " +
719 "display-friendly format";
720 csvFormat = new BooleanArgument('c', "csv", 1, description);
721 parser.addArgument(csvFormat);
722
723 description = "Specifies the seed to use for the random number generator.";
724 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
725 description);
726 randomSeed.addLongIdentifier("random-seed");
727 parser.addArgument(randomSeed);
728
729
730 parser.addDependentArgumentSet(asynchronousMode, ratePerSecond,
731 maxOutstandingRequests);
732 parser.addDependentArgumentSet(maxOutstandingRequests, asynchronousMode);
733
734 parser.addExclusiveArgumentSet(asynchronousMode, simplePageSize);
735 }
736
737
738
739 /**
740 * Indicates whether this tool supports creating connections to multiple
741 * servers. If it is to support multiple servers, then the "--hostname" and
742 * "--port" arguments will be allowed to be provided multiple times, and
743 * will be required to be provided the same number of times. The same type of
744 * communication security and bind credentials will be used for all servers.
745 *
746 * @return {@code true} if this tool supports creating connections to
747 * multiple servers, or {@code false} if not.
748 */
749 @Override()
750 protected boolean supportsMultipleServers()
751 {
752 return true;
753 }
754
755
756
757 /**
758 * Retrieves the connection options that should be used for connections
759 * created for use with this tool.
760 *
761 * @return The connection options that should be used for connections created
762 * for use with this tool.
763 */
764 @Override()
765 public LDAPConnectionOptions getConnectionOptions()
766 {
767 final LDAPConnectionOptions options = new LDAPConnectionOptions();
768 options.setUseSynchronousMode(! asynchronousMode.isPresent());
769 return options;
770 }
771
772
773
774 /**
775 * Performs the actual processing for this tool. In this case, it gets a
776 * connection to the directory server and uses it to perform the requested
777 * searches.
778 *
779 * @return The result code for the processing that was performed.
780 */
781 @Override()
782 public ResultCode doToolProcessing()
783 {
784 runningThread = Thread.currentThread();
785
786 try
787 {
788 return doToolProcessingInternal();
789 }
790 finally
791 {
792 runningThread = null;
793 }
794 }
795
796
797
798 /**
799 * Performs the actual processing for this tool. In this case, it gets a
800 * connection to the directory server and uses it to perform the requested
801 * searches.
802 *
803 * @return The result code for the processing that was performed.
804 */
805 private ResultCode doToolProcessingInternal()
806 {
807 // If the sample rate file argument was specified, then generate the sample
808 // variable rate data file and return.
809 if (sampleRateFile.isPresent())
810 {
811 try
812 {
813 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
814 return ResultCode.SUCCESS;
815 }
816 catch (final Exception e)
817 {
818 debugException(e);
819 err("An error occurred while trying to write sample variable data " +
820 "rate file '", sampleRateFile.getValue().getAbsolutePath(),
821 "': ", getExceptionMessage(e));
822 return ResultCode.LOCAL_ERROR;
823 }
824 }
825
826
827 // Determine the random seed to use.
828 final Long seed;
829 if (randomSeed.isPresent())
830 {
831 seed = Long.valueOf(randomSeed.getValue());
832 }
833 else
834 {
835 seed = null;
836 }
837
838 // Create value patterns for the base DN, filter, and proxied authorization
839 // DN.
840 final ValuePattern dnPattern;
841 try
842 {
843 dnPattern = new ValuePattern(baseDN.getValue(), seed);
844 }
845 catch (final ParseException pe)
846 {
847 debugException(pe);
848 err("Unable to parse the base DN value pattern: ", pe.getMessage());
849 return ResultCode.PARAM_ERROR;
850 }
851
852 final ValuePattern filterPattern;
853 try
854 {
855 filterPattern = new ValuePattern(filter.getValue(), seed);
856 }
857 catch (final ParseException pe)
858 {
859 debugException(pe);
860 err("Unable to parse the filter pattern: ", pe.getMessage());
861 return ResultCode.PARAM_ERROR;
862 }
863
864 final ValuePattern authzIDPattern;
865 if (proxyAs.isPresent())
866 {
867 try
868 {
869 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
870 }
871 catch (final ParseException pe)
872 {
873 debugException(pe);
874 err("Unable to parse the proxied authorization pattern: ",
875 pe.getMessage());
876 return ResultCode.PARAM_ERROR;
877 }
878 }
879 else
880 {
881 authzIDPattern = null;
882 }
883
884 // Get the set of controls to include in search requests.
885 final ArrayList<Control> controlList = new ArrayList<Control>(5);
886 if (assertionFilter.isPresent())
887 {
888 controlList.add(new AssertionRequestControl(assertionFilter.getValue()));
889 }
890
891 if (sortOrder.isPresent())
892 {
893 final ArrayList<SortKey> sortKeys = new ArrayList<SortKey>(5);
894 final StringTokenizer tokenizer =
895 new StringTokenizer(sortOrder.getValue(), ",");
896 while (tokenizer.hasMoreTokens())
897 {
898 String token = tokenizer.nextToken().trim();
899
900 final boolean ascending;
901 if (token.startsWith("+"))
902 {
903 ascending = true;
904 token = token.substring(1);
905 }
906 else if (token.startsWith("-"))
907 {
908 ascending = false;
909 token = token.substring(1);
910 }
911 else
912 {
913 ascending = true;
914 }
915
916 final String attributeName;
917 final String matchingRuleID;
918 final int colonPos = token.indexOf(':');
919 if (colonPos < 0)
920 {
921 attributeName = token;
922 matchingRuleID = null;
923 }
924 else
925 {
926 attributeName = token.substring(0, colonPos);
927 matchingRuleID = token.substring(colonPos+1);
928 }
929
930 sortKeys.add(new SortKey(attributeName, matchingRuleID, (! ascending)));
931 }
932
933 controlList.add(new ServerSideSortRequestControl(sortKeys));
934 }
935
936 if (control.isPresent())
937 {
938 controlList.addAll(control.getValues());
939 }
940
941
942 // Get the attributes to return.
943 final String[] attrs;
944 if (attributes.isPresent())
945 {
946 final List<String> attrList = attributes.getValues();
947 attrs = new String[attrList.size()];
948 attrList.toArray(attrs);
949 }
950 else
951 {
952 attrs = NO_STRINGS;
953 }
954
955
956 // If the --ratePerSecond option was specified, then limit the rate
957 // accordingly.
958 FixedRateBarrier fixedRateBarrier = null;
959 if (ratePerSecond.isPresent() || variableRateData.isPresent())
960 {
961 // We might not have a rate per second if --variableRateData is specified.
962 // The rate typically doesn't matter except when we have warm-up
963 // intervals. In this case, we'll run at the max rate.
964 final int intervalSeconds = collectionInterval.getValue();
965 final int ratePerInterval =
966 (ratePerSecond.getValue() == null)
967 ? Integer.MAX_VALUE
968 : ratePerSecond.getValue() * intervalSeconds;
969 fixedRateBarrier =
970 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
971 }
972
973
974 // If --variableRateData was specified, then initialize a RateAdjustor.
975 RateAdjustor rateAdjustor = null;
976 if (variableRateData.isPresent())
977 {
978 try
979 {
980 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
981 ratePerSecond.getValue(), variableRateData.getValue());
982 }
983 catch (final IOException e)
984 {
985 debugException(e);
986 err("Initializing the variable rates failed: " + e.getMessage());
987 return ResultCode.PARAM_ERROR;
988 }
989 catch (final IllegalArgumentException e)
990 {
991 debugException(e);
992 err("Initializing the variable rates failed: " + e.getMessage());
993 return ResultCode.PARAM_ERROR;
994 }
995 }
996
997
998 // If the --maxOutstandingRequests option was specified, then create the
999 // semaphore used to enforce that limit.
1000 final Semaphore asyncSemaphore;
1001 if (maxOutstandingRequests.isPresent())
1002 {
1003 asyncSemaphore = new Semaphore(maxOutstandingRequests.getValue());
1004 }
1005 else
1006 {
1007 asyncSemaphore = null;
1008 }
1009
1010
1011 // Determine whether to include timestamps in the output and if so what
1012 // format should be used for them.
1013 final boolean includeTimestamp;
1014 final String timeFormat;
1015 if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
1016 {
1017 includeTimestamp = true;
1018 timeFormat = "dd/MM/yyyy HH:mm:ss";
1019 }
1020 else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
1021 {
1022 includeTimestamp = true;
1023 timeFormat = "HH:mm:ss";
1024 }
1025 else
1026 {
1027 includeTimestamp = false;
1028 timeFormat = null;
1029 }
1030
1031
1032 // Determine whether any warm-up intervals should be run.
1033 final long totalIntervals;
1034 final boolean warmUp;
1035 int remainingWarmUpIntervals = warmUpIntervals.getValue();
1036 if (remainingWarmUpIntervals > 0)
1037 {
1038 warmUp = true;
1039 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
1040 }
1041 else
1042 {
1043 warmUp = true;
1044 totalIntervals = 0L + numIntervals.getValue();
1045 }
1046
1047
1048 // Create the table that will be used to format the output.
1049 final OutputFormat outputFormat;
1050 if (csvFormat.isPresent())
1051 {
1052 outputFormat = OutputFormat.CSV;
1053 }
1054 else
1055 {
1056 outputFormat = OutputFormat.COLUMNS;
1057 }
1058
1059 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
1060 timeFormat, outputFormat, " ",
1061 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1062 "Searches/Sec"),
1063 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1064 "Avg Dur ms"),
1065 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1066 "Entries/Srch"),
1067 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1068 "Errors/Sec"),
1069 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1070 "Searches/Sec"),
1071 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1072 "Avg Dur ms"));
1073
1074
1075 // Create values to use for statistics collection.
1076 final AtomicLong searchCounter = new AtomicLong(0L);
1077 final AtomicLong entryCounter = new AtomicLong(0L);
1078 final AtomicLong errorCounter = new AtomicLong(0L);
1079 final AtomicLong searchDurations = new AtomicLong(0L);
1080 final ResultCodeCounter rcCounter = new ResultCodeCounter();
1081
1082
1083 // Determine the length of each interval in milliseconds.
1084 final long intervalMillis = 1000L * collectionInterval.getValue();
1085
1086
1087 // Create the threads to use for the searches.
1088 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
1089 final SearchRateThread[] threads =
1090 new SearchRateThread[numThreads.getValue()];
1091 for (int i=0; i < threads.length; i++)
1092 {
1093 final LDAPConnection connection;
1094 try
1095 {
1096 connection = getConnection();
1097 }
1098 catch (final LDAPException le)
1099 {
1100 debugException(le);
1101 err("Unable to connect to the directory server: ",
1102 getExceptionMessage(le));
1103 return le.getResultCode();
1104 }
1105
1106 threads[i] = new SearchRateThread(this, i, connection,
1107 asynchronousMode.isPresent(), dnPattern, scopeArg.getValue(),
1108 filterPattern, attrs, authzIDPattern, simplePageSize.getValue(),
1109 controlList, iterationsBeforeReconnect.getValue(), barrier,
1110 searchCounter, entryCounter, searchDurations, errorCounter,
1111 rcCounter, fixedRateBarrier, asyncSemaphore);
1112 threads[i].start();
1113 }
1114
1115
1116 // Display the table header.
1117 for (final String headerLine : formatter.getHeaderLines(true))
1118 {
1119 out(headerLine);
1120 }
1121
1122
1123 // Start the RateAdjustor before the threads so that the initial value is
1124 // in place before any load is generated unless we're doing a warm-up in
1125 // which case, we'll start it after the warm-up is complete.
1126 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1127 {
1128 rateAdjustor.start();
1129 }
1130
1131
1132 // Indicate that the threads can start running.
1133 try
1134 {
1135 barrier.await();
1136 }
1137 catch (final Exception e)
1138 {
1139 debugException(e);
1140 }
1141
1142 long overallStartTime = System.nanoTime();
1143 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1144
1145
1146 boolean setOverallStartTime = false;
1147 long lastDuration = 0L;
1148 long lastNumEntries = 0L;
1149 long lastNumErrors = 0L;
1150 long lastNumSearches = 0L;
1151 long lastEndTime = System.nanoTime();
1152 for (long i=0; i < totalIntervals; i++)
1153 {
1154 if (rateAdjustor != null)
1155 {
1156 if (! rateAdjustor.isAlive())
1157 {
1158 out("All of the rates in " + variableRateData.getValue().getName() +
1159 " have been completed.");
1160 break;
1161 }
1162 }
1163
1164 final long startTimeMillis = System.currentTimeMillis();
1165 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1166 nextIntervalStartTime += intervalMillis;
1167 if (sleepTimeMillis > 0)
1168 {
1169 sleeper.sleep(sleepTimeMillis);
1170 }
1171
1172 if (stopRequested.get())
1173 {
1174 break;
1175 }
1176
1177 final long endTime = System.nanoTime();
1178 final long intervalDuration = endTime - lastEndTime;
1179
1180 final long numSearches;
1181 final long numEntries;
1182 final long numErrors;
1183 final long totalDuration;
1184 if (warmUp && (remainingWarmUpIntervals > 0))
1185 {
1186 numSearches = searchCounter.getAndSet(0L);
1187 numEntries = entryCounter.getAndSet(0L);
1188 numErrors = errorCounter.getAndSet(0L);
1189 totalDuration = searchDurations.getAndSet(0L);
1190 }
1191 else
1192 {
1193 numSearches = searchCounter.get();
1194 numEntries = entryCounter.get();
1195 numErrors = errorCounter.get();
1196 totalDuration = searchDurations.get();
1197 }
1198
1199 final long recentNumSearches = numSearches - lastNumSearches;
1200 final long recentNumEntries = numEntries - lastNumEntries;
1201 final long recentNumErrors = numErrors - lastNumErrors;
1202 final long recentDuration = totalDuration - lastDuration;
1203
1204 final double numSeconds = intervalDuration / 1000000000.0d;
1205 final double recentSearchRate = recentNumSearches / numSeconds;
1206 final double recentErrorRate = recentNumErrors / numSeconds;
1207
1208 final double recentAvgDuration;
1209 final double recentEntriesPerSearch;
1210 if (recentNumSearches > 0L)
1211 {
1212 recentEntriesPerSearch = 1.0d * recentNumEntries / recentNumSearches;
1213 recentAvgDuration = 1.0d * recentDuration / recentNumSearches / 1000000;
1214 }
1215 else
1216 {
1217 recentEntriesPerSearch = 0.0d;
1218 recentAvgDuration = 0.0d;
1219 }
1220
1221
1222 if (warmUp && (remainingWarmUpIntervals > 0))
1223 {
1224 out(formatter.formatRow(recentSearchRate, recentAvgDuration,
1225 recentEntriesPerSearch, recentErrorRate, "warming up",
1226 "warming up"));
1227
1228 remainingWarmUpIntervals--;
1229 if (remainingWarmUpIntervals == 0)
1230 {
1231 out("Warm-up completed. Beginning overall statistics collection.");
1232 setOverallStartTime = true;
1233 if (rateAdjustor != null)
1234 {
1235 rateAdjustor.start();
1236 }
1237 }
1238 }
1239 else
1240 {
1241 if (setOverallStartTime)
1242 {
1243 overallStartTime = lastEndTime;
1244 setOverallStartTime = false;
1245 }
1246
1247 final double numOverallSeconds =
1248 (endTime - overallStartTime) / 1000000000.0d;
1249 final double overallSearchRate = numSearches / numOverallSeconds;
1250
1251 final double overallAvgDuration;
1252 if (numSearches > 0L)
1253 {
1254 overallAvgDuration = 1.0d * totalDuration / numSearches / 1000000;
1255 }
1256 else
1257 {
1258 overallAvgDuration = 0.0d;
1259 }
1260
1261 out(formatter.formatRow(recentSearchRate, recentAvgDuration,
1262 recentEntriesPerSearch, recentErrorRate, overallSearchRate,
1263 overallAvgDuration));
1264
1265 lastNumSearches = numSearches;
1266 lastNumEntries = numEntries;
1267 lastNumErrors = numErrors;
1268 lastDuration = totalDuration;
1269 }
1270
1271 final List<ObjectPair<ResultCode,Long>> rcCounts =
1272 rcCounter.getCounts(true);
1273 if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1274 {
1275 err("\tError Results:");
1276 for (final ObjectPair<ResultCode,Long> p : rcCounts)
1277 {
1278 err("\t", p.getFirst().getName(), ": ", p.getSecond());
1279 }
1280 }
1281
1282 lastEndTime = endTime;
1283 }
1284
1285
1286 // Shut down the RateAdjustor if we have one.
1287 if (rateAdjustor != null)
1288 {
1289 rateAdjustor.shutDown();
1290 }
1291
1292
1293 // Stop all of the threads.
1294 ResultCode resultCode = ResultCode.SUCCESS;
1295 for (final SearchRateThread t : threads)
1296 {
1297 t.signalShutdown();
1298 }
1299 for (final SearchRateThread t : threads)
1300 {
1301 final ResultCode r = t.waitForShutdown();
1302 if (resultCode == ResultCode.SUCCESS)
1303 {
1304 resultCode = r;
1305 }
1306 }
1307
1308 return resultCode;
1309 }
1310
1311
1312
1313 /**
1314 * Requests that this tool stop running. This method will attempt to wait
1315 * for all threads to complete before returning control to the caller.
1316 */
1317 public void stopRunning()
1318 {
1319 stopRequested.set(true);
1320 sleeper.wakeup();
1321
1322 final Thread t = runningThread;
1323 if (t != null)
1324 {
1325 try
1326 {
1327 t.join();
1328 }
1329 catch (final Exception e)
1330 {
1331 debugException(e);
1332
1333 if (e instanceof InterruptedException)
1334 {
1335 Thread.currentThread().interrupt();
1336 }
1337 }
1338 }
1339 }
1340
1341
1342
1343 /**
1344 * Retrieves the maximum number of outstanding requests that may be in
1345 * progress at any time, if appropriate.
1346 *
1347 * @return The maximum number of outstanding requests that may be in progress
1348 * at any time, or -1 if the tool was not configured to perform
1349 * asynchronous searches with a maximum number of outstanding
1350 * requests.
1351 */
1352 int getMaxOutstandingRequests()
1353 {
1354 if (maxOutstandingRequests.isPresent())
1355 {
1356 return maxOutstandingRequests.getValue();
1357 }
1358 else
1359 {
1360 return -1;
1361 }
1362 }
1363
1364
1365
1366 /**
1367 * {@inheritDoc}
1368 */
1369 @Override()
1370 public LinkedHashMap<String[],String> getExampleUsages()
1371 {
1372 final LinkedHashMap<String[],String> examples =
1373 new LinkedHashMap<String[],String>(2);
1374
1375 String[] args =
1376 {
1377 "--hostname", "server.example.com",
1378 "--port", "389",
1379 "--bindDN", "uid=admin,dc=example,dc=com",
1380 "--bindPassword", "password",
1381 "--baseDN", "dc=example,dc=com",
1382 "--scope", "sub",
1383 "--filter", "(uid=user.[1-1000000])",
1384 "--attribute", "givenName",
1385 "--attribute", "sn",
1386 "--attribute", "mail",
1387 "--numThreads", "10"
1388 };
1389 String description =
1390 "Test search performance by searching randomly across a set " +
1391 "of one million users located below 'dc=example,dc=com' with ten " +
1392 "concurrent threads. The entries returned to the client will " +
1393 "include the givenName, sn, and mail attributes.";
1394 examples.put(args, description);
1395
1396 args = new String[]
1397 {
1398 "--generateSampleRateFile", "variable-rate-data.txt"
1399 };
1400 description =
1401 "Generate a sample variable rate definition file that may be used " +
1402 "in conjunction with the --variableRateData argument. The sample " +
1403 "file will include comments that describe the format for data to be " +
1404 "included in this file.";
1405 examples.put(args, description);
1406
1407 return examples;
1408 }
1409 }