Master to 4.2.8
[usit-rt.git] / lib / RT / Migrate / Importer.pm
CommitLineData
af59614d
MKG
1# BEGIN BPS TAGGED BLOCK {{{
2#
3# COPYRIGHT:
4#
320f0092 5# This software is Copyright (c) 1996-2014 Best Practical Solutions, LLC
af59614d
MKG
6# <sales@bestpractical.com>
7#
8# (Except where explicitly superseded by other copyright notices)
9#
10#
11# LICENSE:
12#
13# This work is made available to you under the terms of Version 2 of
14# the GNU General Public License. A copy of that license should have
15# been provided with this software, but in any event can be snarfed
16# from www.gnu.org.
17#
18# This work is distributed in the hope that it will be useful, but
19# WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21# General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26# 02110-1301 or visit their web page on the internet at
27# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
28#
29#
30# CONTRIBUTION SUBMISSION POLICY:
31#
32# (The following paragraph is not intended to limit the rights granted
33# to you to modify and distribute this software under the terms of
34# the GNU General Public License and is only of importance to you if
35# you choose to contribute your changes and enhancements to the
36# community by submitting them to Best Practical Solutions, LLC.)
37#
38# By intentionally submitting any modifications, corrections or
39# derivatives to this work, or any other work intended for use with
40# Request Tracker, to Best Practical Solutions, LLC, you confirm that
41# you are the copyright holder for those contributions and you grant
42# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
43# royalty-free, perpetual, license to use, copy, create derivative
44# works based on those contributions, and sublicense and distribute
45# those contributions and any derivatives thereof.
46#
47# END BPS TAGGED BLOCK }}}
48
49package RT::Migrate::Importer;
50
51use strict;
52use warnings;
53
54use Storable qw//;
55use File::Spec;
56use Carp qw/carp/;
57
58sub new {
59 my $class = shift;
60 my $self = bless {}, $class;
61 $self->Init(@_);
62 return $self;
63}
64
65sub Init {
66 my $self = shift;
67 my %args = (
68 OriginalId => undef,
69 Progress => undef,
70 Statefile => undef,
71 DumpObjects => undef,
72 HandleError => undef,
73 @_,
74 );
75
76 # Should we attempt to preserve record IDs as they are created?
77 $self->{OriginalId} = $args{OriginalId};
78
79 $self->{Progress} = $args{Progress};
80
81 $self->{HandleError} = sub { 0 };
82 $self->{HandleError} = $args{HandleError}
83 if $args{HandleError} and ref $args{HandleError} eq 'CODE';
84
85 if ($args{DumpObjects}) {
86 require Data::Dumper;
87 $self->{DumpObjects} = { map { $_ => 1 } @{$args{DumpObjects}} };
88 }
89
90 # Objects we've created
91 $self->{UIDs} = {};
92
93 # Columns we need to update when an object is later created
94 $self->{Pending} = {};
95
96 # Objects missing from the source database before serialization
97 $self->{Invalid} = [];
98
99 # What we created
100 $self->{ObjectCount} = {};
101
102 # To know what global CFs need to be unglobal'd and applied to what
103 $self->{NewQueues} = [];
104 $self->{NewCFs} = [];
105}
106
107sub Metadata {
108 my $self = shift;
109 return $self->{Metadata};
110}
111
112sub LoadMetadata {
113 my $self = shift;
114 my ($data) = @_;
115
116 return if $self->{Metadata};
117 $self->{Metadata} = $data;
118
119 die "Incompatible format version: ".$data->{Format}
120 if $data->{Format} ne "0.8";
121
122 $self->{Organization} = $data->{Organization};
123 $self->{Clone} = $data->{Clone};
124 $self->{Incremental} = $data->{Incremental};
125 $self->{Files} = $data->{Files} if $data->{Final};
126}
127
128sub InitStream {
129 my $self = shift;
130
131 die "Stream initialized after objects have been recieved!"
132 if keys %{ $self->{UIDs} };
133
134 die "Cloning does not support importing the Original Id separately\n"
135 if $self->{OriginalId} and $self->{Clone};
136
137 die "RT already contains data; overwriting will not work\n"
138 if ($self->{Clone} and not $self->{Incremental})
139 and RT->SystemUser->Id;
140
141 # Basic facts of life, as a safety net
142 $self->Resolve( RT->System->UID => ref RT->System, RT->System->Id );
143 $self->SkipTransactions( RT->System->UID );
144
145 if ($self->{OriginalId}) {
146 # Where to shove the original ticket ID
147 my $cf = RT::CustomField->new( RT->SystemUser );
c33a4027 148 $cf->LoadByName( Name => $self->{OriginalId}, LookupType => RT::Ticket->CustomFieldLookupType, ObjectId => 0 );
af59614d
MKG
149 unless ($cf->Id) {
150 warn "Failed to find global CF named $self->{OriginalId} -- creating one";
151 $cf->Create(
152 Queue => 0,
153 Name => $self->{OriginalId},
154 Type => 'FreeformSingle',
155 );
156 }
157 }
158}
159
160sub Resolve {
161 my $self = shift;
162 my ($uid, $class, $id) = @_;
163 $self->{UIDs}{$uid} = [ $class, $id ];
164 return unless $self->{Pending}{$uid};
165
166 for my $ref (@{$self->{Pending}{$uid}}) {
167 my ($pclass, $pid) = @{ $self->Lookup( $ref->{uid} ) };
168 my $obj = $pclass->new( RT->SystemUser );
169 $obj->LoadByCols( Id => $pid );
170 $obj->__Set(
171 Field => $ref->{column},
172 Value => $id,
173 ) if defined $ref->{column};
174 $obj->__Set(
175 Field => $ref->{classcolumn},
176 Value => $class,
177 ) if defined $ref->{classcolumn};
178 $obj->__Set(
179 Field => $ref->{uri},
180 Value => $self->LookupObj($uid)->URI,
181 ) if defined $ref->{uri};
182 }
183 delete $self->{Pending}{$uid};
184}
185
186sub Lookup {
187 my $self = shift;
188 my ($uid) = @_;
189 unless (defined $uid) {
190 carp "Tried to lookup an undefined UID";
191 return;
192 }
193 return $self->{UIDs}{$uid};
194}
195
196sub LookupObj {
197 my $self = shift;
198 my ($uid) = @_;
199 my $ref = $self->Lookup( $uid );
200 return unless $ref;
201 my ($class, $id) = @{ $ref };
202
203 my $obj = $class->new( RT->SystemUser );
204 $obj->Load( $id );
205 return $obj;
206}
207
208sub Postpone {
209 my $self = shift;
210 my %args = (
211 for => undef,
212 uid => undef,
213 column => undef,
214 classcolumn => undef,
215 uri => undef,
216 @_,
217 );
218 my $uid = delete $args{for};
219
220 if (defined $uid) {
221 push @{$self->{Pending}{$uid}}, \%args;
222 } else {
223 push @{$self->{Invalid}}, \%args;
224 }
225}
226
227sub SkipTransactions {
228 my $self = shift;
229 my ($uid) = @_;
230 return if $self->{Clone};
231 $self->{SkipTransactions}{$uid} = 1;
232}
233
234sub ShouldSkipTransaction {
235 my $self = shift;
236 my ($uid) = @_;
237 return exists $self->{SkipTransactions}{$uid};
238}
239
240sub MergeValues {
241 my $self = shift;
242 my ($obj, $data) = @_;
243 for my $col (keys %{$data}) {
244 next if defined $obj->__Value($col) and length $obj->__Value($col);
245 next unless defined $data->{$col} and length $data->{$col};
246
247 if (ref $data->{$col}) {
248 my $uid = ${ $data->{$col} };
249 my $ref = $self->Lookup( $uid );
250 if ($ref) {
251 $data->{$col} = $ref->[1];
252 } else {
253 $self->Postpone(
254 for => $obj->UID,
255 uid => $uid,
256 column => $col,
257 );
258 next;
259 }
260 }
261 $obj->__Set( Field => $col, Value => $data->{$col} );
262 }
263}
264
265sub SkipBy {
266 my $self = shift;
267 my ($column, $class, $uid, $data) = @_;
268
269 my $obj = $class->new( RT->SystemUser );
270 $obj->Load( $data->{$column} );
271 return unless $obj->Id;
272
273 $self->SkipTransactions( $uid );
274
275 $self->Resolve( $uid => $class => $obj->Id );
276 return $obj;
277}
278
279sub MergeBy {
280 my $self = shift;
281 my ($column, $class, $uid, $data) = @_;
282
283 my $obj = $self->SkipBy(@_);
284 return unless $obj;
285 $self->MergeValues( $obj, $data );
286 return 1;
287}
288
289sub Qualify {
290 my $self = shift;
291 my ($string) = @_;
292 return $string if $self->{Clone};
293 return $string if not defined $self->{Organization};
c33a4027 294 return $string if $self->{Organization} eq $RT::Organization;
af59614d
MKG
295 return $self->{Organization}.": $string";
296}
297
298sub Create {
299 my $self = shift;
300 my ($class, $uid, $data) = @_;
301
302 # Use a simpler pre-inflation if we're cloning
303 if ($self->{Clone}) {
304 $class->RT::Record::PreInflate( $self, $uid, $data );
305 } else {
306 # Non-cloning always wants to make its own id
307 delete $data->{id};
308 return unless $class->PreInflate( $self, $uid, $data );
309 }
310
311 my $obj = $class->new( RT->SystemUser );
312 my ($id, $msg) = eval {
313 # catch and rethrow on the outside so we can provide more info
314 local $SIG{__DIE__};
315 $obj->DBIx::SearchBuilder::Record::Create(
316 %{$data}
317 );
318 };
319 if (not $id or $@) {
320 $msg ||= ''; # avoid undef
321 my $err = "Failed to create $uid: $msg $@\n" . Data::Dumper::Dumper($data) . "\n";
322 if (not $self->{HandleError}->($self, $err)) {
323 die $err;
324 } else {
325 return;
326 }
327 }
328
329 $self->{ObjectCount}{$class}++;
330 $self->Resolve( $uid => $class, $id );
331
332 # Load it back to get real values into the columns
333 $obj = $class->new( RT->SystemUser );
334 $obj->Load( $id );
335 $obj->PostInflate( $self );
336
337 return $obj;
338}
339
340sub ReadStream {
341 my $self = shift;
342 my ($fh) = @_;
343
344 no warnings 'redefine';
345 local *RT::Ticket::Load = sub {
346 my $self = shift;
347 my $id = shift;
348 $self->LoadById( $id );
349 return $self->Id;
350 };
351
352 my $loaded = Storable::fd_retrieve($fh);
353
354 # Metadata is stored at the start of the stream as a hashref
355 if (ref $loaded eq "HASH") {
356 $self->LoadMetadata( $loaded );
357 $self->InitStream;
358 return;
359 }
360
361 my ($class, $uid, $data) = @{$loaded};
362
363 if ($self->{Incremental}) {
364 my $obj = $class->new( RT->SystemUser );
365 $obj->Load( $data->{id} );
366 if (not $uid) {
367 # undef $uid means "delete it"
368 $obj->Delete;
369 $self->{ObjectCount}{$class}++;
370 } elsif ( $obj->Id ) {
371 # If it exists, update it
372 $class->RT::Record::PreInflate( $self, $uid, $data );
373 $obj->__Set( Field => $_, Value => $data->{$_} )
374 for keys %{ $data };
375 $self->{ObjectCount}{$class}++;
376 } else {
377 # Otherwise, make it
378 $obj = $self->Create( $class, $uid, $data );
379 }
380 $self->{Progress}->($obj) if $obj and $self->{Progress};
381 return;
382 } elsif ($self->{Clone}) {
383 my $obj = $self->Create( $class, $uid, $data );
384 $self->{Progress}->($obj) if $obj and $self->{Progress};
385 return;
386 }
387
388 # If it's a queue, store its ID away, as we'll need to know
389 # it to split global CFs into non-global across those
390 # fields. We do this before inflating, so that queues which
391 # got merged still get the CFs applied
392 push @{$self->{NewQueues}}, $uid
393 if $class eq "RT::Queue";
394
395 my $origid = $data->{id};
396 my $obj = $self->Create( $class, $uid, $data );
397 return unless $obj;
398
399 # If it's a ticket, we might need to create a
400 # TicketCustomField for the previous ID
401 if ($class eq "RT::Ticket" and $self->{OriginalId}) {
402 my ($id, $msg) = $obj->AddCustomFieldValue(
403 Field => $self->{OriginalId},
404 Value => $self->Organization . ":$origid",
405 RecordTransaction => 0,
406 );
407 warn "Failed to add custom field to $uid: $msg"
408 unless $id;
409 }
410
411 # If it's a CF, we don't know yet if it's global (the OCF
412 # hasn't been created yet) to store away the CF for later
413 # inspection
414 push @{$self->{NewCFs}}, $uid
415 if $class eq "RT::CustomField"
416 and $obj->LookupType =~ /^RT::Queue/;
417
418 $self->{Progress}->($obj) if $self->{Progress};
419}
420
421sub CloseStream {
422 my $self = shift;
423
424 $self->{Progress}->(undef, 'force') if $self->{Progress};
425
426 return if $self->{Clone};
427
428 # Take global CFs which we made and make them un-global
429 my @queues = grep {$_} map {$self->LookupObj( $_ )} @{$self->{NewQueues}};
430 for my $obj (map {$self->LookupObj( $_ )} @{$self->{NewCFs}}) {
320f0092 431 my $ocf = $obj->IsGlobal or next;
af59614d
MKG
432 $ocf->Delete;
433 $obj->AddToObject( $_ ) for @queues;
434 }
435 $self->{NewQueues} = [];
436 $self->{NewCFs} = [];
437}
438
439
440sub ObjectCount {
441 my $self = shift;
442 return %{ $self->{ObjectCount} };
443}
444
445sub Missing {
446 my $self = shift;
447 return wantarray ? sort keys %{ $self->{Pending} }
448 : keys %{ $self->{Pending} };
449}
450
451sub Invalid {
452 my $self = shift;
453 return wantarray ? sort { $a->{uid} cmp $b->{uid} } @{ $self->{Invalid} }
454 : $self->{Invalid};
455}
456
457sub Organization {
458 my $self = shift;
459 return $self->{Organization};
460}
461
462sub Progress {
463 my $self = shift;
464 return defined $self->{Progress} unless @_;
465 return $self->{Progress} = $_[0];
466}
467
4681;