Master to 4.2.8
[usit-rt.git] / sbin / rt-serializer
CommitLineData
320f0092
MKG
1#!/usr/bin/perl
2# BEGIN BPS TAGGED BLOCK {{{
3#
4# COPYRIGHT:
5#
6# This software is Copyright (c) 1996-2014 Best Practical Solutions, LLC
7# <sales@bestpractical.com>
8#
9# (Except where explicitly superseded by other copyright notices)
10#
11#
12# LICENSE:
13#
14# This work is made available to you under the terms of Version 2 of
15# the GNU General Public License. A copy of that license should have
16# been provided with this software, but in any event can be snarfed
17# from www.gnu.org.
18#
19# This work is distributed in the hope that it will be useful, but
20# WITHOUT ANY WARRANTY; without even the implied warranty of
21# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22# General Public License for more details.
23#
24# You should have received a copy of the GNU General Public License
25# along with this program; if not, write to the Free Software
26# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
27# 02110-1301 or visit their web page on the internet at
28# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
29#
30#
31# CONTRIBUTION SUBMISSION POLICY:
32#
33# (The following paragraph is not intended to limit the rights granted
34# to you to modify and distribute this software under the terms of
35# the GNU General Public License and is only of importance to you if
36# you choose to contribute your changes and enhancements to the
37# community by submitting them to Best Practical Solutions, LLC.)
38#
39# By intentionally submitting any modifications, corrections or
40# derivatives to this work, or any other work intended for use with
41# Request Tracker, to Best Practical Solutions, LLC, you confirm that
42# you are the copyright holder for those contributions and you grant
43# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
44# royalty-free, perpetual, license to use, copy, create derivative
45# works based on those contributions, and sublicense and distribute
46# those contributions and any derivatives thereof.
47#
48# END BPS TAGGED BLOCK }}}
49use strict;
50use warnings;
51
52# fix lib paths, some may be relative
53BEGIN {
54 require File::Spec;
55 my @libs = ("lib", "local/lib");
56 my $bin_path;
57
58 for my $lib (@libs) {
59 unless ( File::Spec->file_name_is_absolute($lib) ) {
60 unless ($bin_path) {
61 if ( File::Spec->file_name_is_absolute(__FILE__) ) {
62 $bin_path = ( File::Spec->splitpath(__FILE__) )[1];
63 }
64 else {
65 require FindBin;
66 no warnings "once";
67 $bin_path = $FindBin::Bin;
68 }
69 }
70 $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
71 }
72 unshift @INC, $lib;
73 }
74
75}
76
77use RT;
78RT::LoadConfig();
79RT->Config->Set(RecordBaseClass => "DBIx::SearchBuilder::Record");
80RT::Init();
81
82use RT::Migrate;
83use RT::Migrate::Serializer::File;
84use Getopt::Long;
85use Pod::Usage qw//;
86use Time::HiRes qw//;
87
88my %OPT;
89GetOptions(
90 \%OPT,
91 "help|?",
92 "verbose|v!",
93 "quiet|q!",
94
95 "directory|d=s",
96 "force|f!",
97 "size|s=i",
98
99 "users!",
100 "groups!",
101 "deleted!",
102
103 "scrips!",
104 "tickets!",
105 "acls!",
106
107 "clone",
108 "incremental",
109
110 "gc=i",
111 "page=i",
112) or Pod::Usage::pod2usage();
113
114Pod::Usage::pod2usage(-verbose => 1) if $OPT{help};
115
116my %args;
117$args{Directory} = $OPT{directory};
118$args{Force} = $OPT{force};
119$args{MaxFileSize} = $OPT{size} if $OPT{size};
120
121$args{AllUsers} = $OPT{users} if defined $OPT{users};
122$args{AllGroups} = $OPT{groups} if defined $OPT{groups};
123$args{FollowDeleted} = $OPT{deleted} if defined $OPT{deleted};
124
125$args{FollowScrips} = $OPT{scrips} if defined $OPT{scrips};
126$args{FollowTickets} = $OPT{tickets} if defined $OPT{tickets};
127$args{FollowACL} = $OPT{acls} if defined $OPT{acls};
128
129$args{Clone} = $OPT{clone} if $OPT{clone};
130$args{Incremental} = $OPT{incremental} if $OPT{incremental};
131
132$args{GC} = defined $OPT{gc} ? $OPT{gc} : 5000;
133$args{Page} = defined $OPT{page} ? $OPT{page} : 100;
134
135if (($OPT{clone} or $OPT{incremental})
136 and grep { /^(users|groups|deleted|scrips|tickets|acls)$/ } keys %OPT) {
137 die "You cannot specify object types when cloning.\n\nPlease see $0 --help.\n";
138}
139
140my $walker;
141
142my $gnuplot = `which gnuplot`;
143my $msg = "";
144if (-t STDOUT and not $OPT{verbose} and not $OPT{quiet}) {
145 $args{Progress} = RT::Migrate::progress(
146 top => \&gnuplot,
147 bottom => sub { print "\n$msg"; $msg = ""; },
148 counts => sub { $walker->ObjectCount },
149 max => { estimate() },
150 );
151 $args{MessageHandler} = sub {
152 print "\r", " "x60, "\r", $_[-1]; $msg = $_[-1];
153 };
154 $args{Verbose} = 0;
155}
156$args{Verbose} = 0 if $OPT{quiet};
157
158
159$walker = RT::Migrate::Serializer::File->new( %args );
160
161my $log = RT::Migrate::setup_logging( $walker->{Directory} => 'serializer.log' );
162print "Logging warnings and errors to $log\n" if $log;
163
164print "Beginning database serialization...";
165my %counts = $walker->Export;
166
167my @files = $walker->Files;
168print "Wrote @{[scalar @files]} files:\n";
169print " $_\n" for @files;
170print "\n";
171
172print "Total object counts:\n";
173for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
174 printf "%8d %s\n", $counts{$_}, $_;
175}
176
177if ($log and -s $log) {
178 print STDERR "\n! Some warnings or errors occurred during serialization."
179 ."\n! Please see $log for details.\n\n";
180} else {
181 unlink $log;
182}
183
184sub estimate {
185 $| = 1;
186 my %e;
187
188 # Expected types we'll serialize
189 my @types = map {"RT::$_"} qw/
190 Queue Ticket Transaction Attachment Link
191 User Group GroupMember Attribute
192 CustomField CustomFieldValue
193 ObjectCustomField ObjectCustomFieldValue
194 /;
195
196 for my $class (@types) {
197 print "Estimating $class count...";
198 my $collection = $class . "s";
c33a4027 199 if ($collection->require) {
320f0092
MKG
200 my $objs = $collection->new( RT->SystemUser );
201 $objs->FindAllRows;
202 $objs->UnLimit;
203 $objs->{allow_deleted_search} = 1 if $class eq "RT::Ticket";
204 $e{$class} = $objs->DBIx::SearchBuilder::Count;
205 }
206 print "\r", " "x60, "\r";
207 }
208
209 return %e;
210}
211
212
213sub gnuplot {
214 my ($elapsed, $rows, $cols) = @_;
215 my $length = $walker->StackSize;
216 my $file = $walker->Directory . "/progress.plot";
217 open(my $dat, ">>", $file);
218 printf $dat "%10.3f\t%8d\n", $elapsed, $length;
219 close $dat;
220
221 if ($rows <= 24 or not $gnuplot) {
222 print "\n\n";
223 } elsif ($elapsed) {
224 my $gnuplot = qx|
225 gnuplot -e '
226 set term dumb $cols @{[$rows - 12]};
227 set xlabel "Seconds";
228 unset key;
229 set xrange [0:*];
230 set yrange [0:*];
231 set title "Queue length";
232 plot "$file" using 1:2 with lines
233 '
234 |;
235 if ($? == 0 and $gnuplot) {
236 $gnuplot =~ s/^(\s*\n)//;
237 print $gnuplot;
238 unlink $file;
239 } else {
240 warn "Couldn't run gnuplot (\$? == $?): $!\n";
241 }
242 } else {
243 print "\n" for 1..($rows - 13);
244 }
245}
246
247=head1 NAME
248
249rt-serializer - Serialize an RT database to disk
250
251=head1 SYNOPSIS
252
253 rt-validator --check && rt-serializer
254
255This script is used to write out the entire RT database to disk, for
256later import into a different RT instance. It requires that the data in
257the database be self-consistent, in order to do so; please make sure
258that the database being exported passes validation by L<rt-validator>
259before attempting to use C<rt-serializer>.
260
261While running, it will attempt to estimate the number of remaining
262objects to be serialized; these estimates are pessimistic, and will be
263incorrect if C<--no-users>, C<--no-groups>, or C<--no-tickets> are used.
264
265If the controlling terminal is large enough (more than 25 columns high)
266and the C<gnuplot> program is installed, it will also show a textual
267graph of the queue size over time.
268
269=head2 OPTIONS
270
271=over
272
273=item B<--directory> I<name>
274
275The name of the output directory to write data files to, which should
276not exist yet; it is a fatal error if it does. Defaults to
277C<< ./I<$Organization>:I<Date>/ >>, where I<$Organization> is as set in
278F<RT_SiteConfig.pm>, and I<Date> is today's date.
279
280=item B<--force>
281
282Remove the output directory before starting.
283
284=item B<--size> I<megabytes>
285
286By default, C<rt-serializer> chunks its output into data files which are
287around 32Mb in size; this option is used to set a different threshold
288size, in megabytes. Note that this is the threshold after which it
289rotates to writing a new file, and is as such the I<lower bound> on the
290size of each output file.
291
292=item B<--no-users>
293
294By default, all privileged users are serialized; passing C<--no-users>
c33a4027
MKG
295limits it to only those users which are referenced by serialized tickets
296and history, and are thus necessary for internal consistency.
320f0092
MKG
297
298=item B<--no-groups>
299
300By default, all groups are serialized; passing C<--no-groups> limits it
c33a4027
MKG
301to only system-internal groups, which are needed for internal
302consistency.
320f0092
MKG
303
304=item B<--no-deleted>
305
306By default, all tickets, including deleted tickets, are serialized;
307passing C<--no-deleted> skips deleted tickets during serialization.
308
309=item B<--scrips>
310
311No scrips or templates are serialized by default; this option forces all
312scrips and templates to be serialized.
313
314=item B<--acls>
315
316No ACLs are serialized by default; this option forces all ACLs to be
317serialized.
318
319=item B<--no-tickets>
320
321Skip serialization of all ticket data.
322
323=item B<--clone>
324
325Serializes your entire database, creating a clone. This option should
326be used if you want to migrate your RT database from one database type
327to another (e.g. MySQL to Postgres). It is an error to combine
328C<--clone> with any option that limits object types serialized. No
329dependency walking is performed when cloning. C<rt-importer> will detect
330that your serialized data set was generated by a clone.
331
332=item B<--incremental>
333
334Will generate an incremenal serialized dataset using the data stored in
335your IncrementalRecords database table. This assumes that you have created
336that table and run RT using the Record_Local.pm shim as documented in
337C<docs/incremental-export/>.
338
339=item B<--gc> I<n>
340
341Adjust how often the garbage collection sweep is done; lower numbers are
342more frequent. See L</GARBAGE COLLECTION>.
343
344=item B<--page> I<n>
345
346Adjust how many rows are pulled from the database in a single query. Disable
347paging by setting this to 0. Defaults to 100.
348
349Keep in mind that rows from RT's Attachments table are the limiting factor when
350determining page size. You should likely be aiming for 60-75% of your total
351memory on an otherwise unloaded box.
352
353=item B<--quiet>
354
355Do not show graphical progress UI.
356
357=item B<--verbose>
358
359Do not show graphical progress UI, but rather log was each row is
360written out.
361
362=back
363
364=head1 GARBAGE COLLECTION
365
366C<rt-serializer> maintains a priority queue of objects to serialize, or
367searches which may result in objects to serialize. When inserting into
368this queue, it does no checking if the object in question is already in
369the queue, or if the search will contain any results. These checks are
370done when the object reaches the front of the queue, or during periodic
371garbage collection.
372
373During periodic garbage collection, the entire queue is swept for
374objects which have already been serialized, occur more than once in the
375queue, and searches which contain no results in the database. This is
376done to reduce the memory footprint of the serialization process, and is
377triggered when enough new objects have been placed in the queue. This
378parameter is tunable via the C<--gc> parameter, which defaults to
379running garbage collection every 5,000 objects inserted into the queue;
380smaller numbers will result in more frequent garbage collection.
381
382The default of 5,000 is roughly tuned based on a database with several
383thousand tickets, but optimal values will vary wildly depending on
384database configuration and size. Values as low as 25 have provided
385speedups with smaller databases; if speed is a factor, experimenting
386with different C<--gc> values may be helpful. Note that there are
387significant boundary condition changes in serialization rate, as the
388queue empties and fills, causing the time estimates to be rather
389imprecise near the start and end of the process.
390
391Setting C<--gc> to 0 turns off all garbage collection. Be aware that
392this will bloat the memory usage of the serializer. Any negative value
393for C<--gc> turns off periodic garbage collection and instead objects
394already serialized or in the queue are checked for at the time they
395would be inserted.
396
397=cut
398