Master to 4.2.8
[usit-rt.git] / lib / RT / Migrate / Serializer.pm
CommitLineData
af59614d
MKG
1# BEGIN BPS TAGGED BLOCK {{{
2#
3# COPYRIGHT:
4#
320f0092 5# This software is Copyright (c) 1996-2014 Best Practical Solutions, LLC
af59614d
MKG
6# <sales@bestpractical.com>
7#
8# (Except where explicitly superseded by other copyright notices)
9#
10#
11# LICENSE:
12#
13# This work is made available to you under the terms of Version 2 of
14# the GNU General Public License. A copy of that license should have
15# been provided with this software, but in any event can be snarfed
16# from www.gnu.org.
17#
18# This work is distributed in the hope that it will be useful, but
19# WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21# General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26# 02110-1301 or visit their web page on the internet at
27# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
28#
29#
30# CONTRIBUTION SUBMISSION POLICY:
31#
32# (The following paragraph is not intended to limit the rights granted
33# to you to modify and distribute this software under the terms of
34# the GNU General Public License and is only of importance to you if
35# you choose to contribute your changes and enhancements to the
36# community by submitting them to Best Practical Solutions, LLC.)
37#
38# By intentionally submitting any modifications, corrections or
39# derivatives to this work, or any other work intended for use with
40# Request Tracker, to Best Practical Solutions, LLC, you confirm that
41# you are the copyright holder for those contributions and you grant
42# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
43# royalty-free, perpetual, license to use, copy, create derivative
44# works based on those contributions, and sublicense and distribute
45# those contributions and any derivatives thereof.
46#
47# END BPS TAGGED BLOCK }}}
48
49package RT::Migrate::Serializer;
50
51use strict;
52use warnings;
53
54use base 'RT::DependencyWalker';
55
56use Storable qw//;
57sub cmp_version($$) { RT::Handle::cmp_version($_[0],$_[1]) };
58use RT::Migrate::Incremental;
59use RT::Migrate::Serializer::IncrementalRecord;
60use RT::Migrate::Serializer::IncrementalRecords;
61
62sub Init {
63 my $self = shift;
64
65 my %args = (
66 AllUsers => 1,
67 AllGroups => 1,
68 FollowDeleted => 1,
69
70 FollowScrips => 0,
71 FollowTickets => 1,
72 FollowACL => 0,
73
74 Clone => 0,
75 Incremental => 0,
76
77 Verbose => 1,
78 @_,
79 );
80
81 $self->{Verbose} = delete $args{Verbose};
82
83 $self->{$_} = delete $args{$_}
84 for qw/
85 AllUsers
86 AllGroups
87 FollowDeleted
88 FollowScrips
89 FollowTickets
90 FollowACL
91 Clone
92 Incremental
93 /;
94
95 $self->{Clone} = 1 if $self->{Incremental};
96
97 $self->SUPER::Init(@_, First => "top");
98
99 # Keep track of the number of each type of object written out
100 $self->{ObjectCount} = {};
101
102 if ($self->{Clone}) {
103 $self->PushAll;
104 } else {
105 $self->PushBasics;
106 }
107}
108
109sub Metadata {
110 my $self = shift;
111
112 # Determine the highest upgrade step that we run
113 my @versions = ($RT::VERSION, keys %RT::Migrate::Incremental::UPGRADES);
114 my ($max) = reverse sort cmp_version @versions;
320f0092
MKG
115 # we don't want to run upgrades to 4.2.x if we're running
116 # the serializier on an 4.0 instance.
117 $max = $RT::VERSION unless $self->{Incremental};
af59614d
MKG
118
119 return {
120 Format => "0.8",
121 VersionFrom => $RT::VERSION,
122 Version => $max,
123 Organization => $RT::Organization,
124 Clone => $self->{Clone},
125 Incremental => $self->{Incremental},
126 ObjectCount => { $self->ObjectCount },
127 @_,
128 },
129}
130
131sub PushAll {
132 my $self = shift;
133
134 # To keep unique constraints happy, we need to remove old records
135 # before we insert new ones. This fixes the case where a
136 # GroupMember was deleted and re-added (with a new id, but the same
137 # membership).
138 if ($self->{Incremental}) {
139 my $removed = RT::Migrate::Serializer::IncrementalRecords->new( RT->SystemUser );
140 $removed->Limit( FIELD => "UpdateType", VALUE => 3 );
141 $removed->OrderBy( FIELD => 'id' );
142 $self->PushObj( $removed );
143 }
144 # XXX: This is sadly not sufficient to deal with the general case of
145 # non-id unique constraints, such as queue names. If queues A and B
146 # existed, and B->C and A->B renames were done, these will be
147 # serialized with A->B first, which will fail because there already
148 # exists a B.
149
150 # Principals first; while we don't serialize these separately during
151 # normal dependency walking (we fold them into users and groups),
152 # having them separate during cloning makes logic simpler.
153 $self->PushCollections(qw(Principals));
154
155 # Users and groups
156 $self->PushCollections(qw(Users Groups GroupMembers));
157
158 # Tickets
159 $self->PushCollections(qw(Queues Tickets Transactions Attachments Links));
160
161 # Articles
162 $self->PushCollections(qw(Articles), map { ($_, "Object$_") } qw(Classes Topics));
163
164 # Custom Fields
c33a4027 165 if (RT::ObjectCustomFields->require) {
af59614d 166 $self->PushCollections(map { ($_, "Object$_") } qw(CustomFields CustomFieldValues));
c33a4027 167 } elsif (RT::TicketCustomFieldValues->require) {
af59614d
MKG
168 $self->PushCollections(qw(CustomFields CustomFieldValues TicketCustomFieldValues));
169 }
170
171 # ACLs
172 $self->PushCollections(qw(ACL));
173
174 # Scrips
c33a4027 175 $self->PushCollections(qw(Scrips ObjectScrips ScripActions ScripConditions Templates));
af59614d
MKG
176
177 # Attributes
178 $self->PushCollections(qw(Attributes));
179}
180
181sub PushCollections {
182 my $self = shift;
183
184 for my $type (@_) {
185 my $class = "RT::\u$type";
186
c33a4027 187 $class->require or next;
af59614d
MKG
188 my $collection = $class->new( RT->SystemUser );
189 $collection->FindAllRows; # be explicit
190 $collection->CleanSlate; # some collections (like groups and users) join in _Init
191 $collection->UnLimit;
192 $collection->OrderBy( FIELD => 'id' );
193
194 if ($self->{Clone}) {
195 if ($collection->isa('RT::Tickets')) {
196 $collection->{allow_deleted_search} = 1;
197 $collection->IgnoreType; # looking_at_type
198 }
199 elsif ($collection->isa('RT::ObjectCustomFieldValues')) {
200 # FindAllRows (find_disabled_rows) isn't used by OCFVs
201 $collection->{find_expired_rows} = 1;
202 }
203
204 if ($self->{Incremental}) {
205 my $alias = $collection->Join(
206 ALIAS1 => "main",
207 FIELD1 => "id",
208 TABLE2 => "IncrementalRecords",
209 FIELD2 => "ObjectId",
210 );
211 $collection->DBIx::SearchBuilder::Limit(
212 ALIAS => $alias,
213 FIELD => "ObjectType",
214 VALUE => ref($collection->NewItem),
215 );
216 }
217 }
218
219 $self->PushObj( $collection );
220 }
221}
222
223sub PushBasics {
224 my $self = shift;
225
226 # System users
227 for my $name (qw/RT_System root nobody/) {
228 my $user = RT::User->new( RT->SystemUser );
229 my ($id, $msg) = $user->Load( $name );
230 warn "No '$name' user found: $msg" unless $id;
231 $self->PushObj( $user ) if $id;
232 }
233
234 # System groups
235 foreach my $name (qw(Everyone Privileged Unprivileged)) {
236 my $group = RT::Group->new( RT->SystemUser );
237 my ($id, $msg) = $group->LoadSystemInternalGroup( $name );
238 warn "No '$name' group found: $msg" unless $id;
239 $self->PushObj( $group ) if $id;
240 }
241
242 # System role groups
243 my $systemroles = RT::Groups->new( RT->SystemUser );
320f0092 244 $systemroles->LimitToRolesForObject( RT->System );
af59614d
MKG
245 $self->PushObj( $systemroles );
246
247 # CFs on Users, Groups, Queues
248 my $cfs = RT::CustomFields->new( RT->SystemUser );
249 $cfs->Limit(
250 FIELD => 'LookupType',
251 VALUE => $_
252 ) for qw/RT::User RT::Group RT::Queue/;
253 $self->PushObj( $cfs );
254
255 # Global attributes
256 my $attributes = RT::Attributes->new( RT->SystemUser );
257 $attributes->LimitToObject( $RT::System );
258 $self->PushObj( $attributes );
259
260 # Global ACLs
261 if ($self->{FollowACL}) {
262 my $acls = RT::ACL->new( RT->SystemUser );
263 $acls->LimitToObject( $RT::System );
264 $self->PushObj( $acls );
265 }
266
267 # Global scrips
268 if ($self->{FollowScrips}) {
269 my $scrips = RT::Scrips->new( RT->SystemUser );
270 $scrips->LimitToGlobal;
271
272 my $templates = RT::Templates->new( RT->SystemUser );
273 $templates->LimitToGlobal;
274
275 $self->PushObj( $scrips, $templates );
276 $self->PushCollections(qw(ScripActions ScripConditions));
277 }
278
279 if ($self->{AllUsers}) {
280 my $users = RT::Users->new( RT->SystemUser );
281 $users->LimitToPrivileged;
282 $self->PushObj( $users );
283 }
284
285 if ($self->{AllGroups}) {
286 my $groups = RT::Groups->new( RT->SystemUser );
287 $groups->LimitToUserDefinedGroups;
288 $self->PushObj( $groups );
289 }
290
c33a4027 291 if (RT::Articles->require) {
af59614d
MKG
292 $self->PushCollections(qw(Topics Classes));
293 }
294
295 $self->PushCollections(qw(Queues));
296}
297
298sub InitStream {
299 my $self = shift;
300
301 # Write the initial metadata
302 my $meta = $self->Metadata;
303 $! = 0;
304 Storable::nstore_fd( $meta, $self->{Filehandle} );
305 die "Failed to write metadata: $!" if $!;
306
307 return unless cmp_version($meta->{VersionFrom}, $meta->{Version}) < 0;
308
309 my %transforms;
310 for my $v (sort cmp_version keys %RT::Migrate::Incremental::UPGRADES) {
311 for my $ref (keys %{$RT::Migrate::Incremental::UPGRADES{$v}}) {
312 push @{$transforms{$ref}}, $RT::Migrate::Incremental::UPGRADES{$v}{$ref};
313 }
314 }
315 for my $ref (keys %transforms) {
316 # XXX Does not correctly deal with updates of $classref, which
317 # should technically apply all later transforms of the _new_
318 # class. This is not relevant in the current upgrades, as
319 # RT::ObjectCustomFieldValues do not have interesting later
320 # upgrades if you start from 3.2 (which does
321 # RT::TicketCustomFieldValues -> RT::ObjectCustomFieldValues)
322 $self->{Transform}{$ref} = sub {
323 my ($dat, $classref) = @_;
324 my @extra;
325 for my $c (@{$transforms{$ref}}) {
326 push @extra, $c->($dat, $classref);
327 return @extra if not $$classref;
328 }
329 return @extra;
330 };
331 }
332}
333
334sub NextPage {
335 my $self = shift;
336 my ($collection, $last) = @_;
337
338 $last ||= 0;
339
340 if ($self->{Clone}) {
341 # Clone provides guaranteed ordering by id and with no other id limits
342 # worry about trampling
343
344 # Use DBIx::SearchBuilder::Limit explicitly to avoid shenanigans in RT::Tickets
345 $collection->DBIx::SearchBuilder::Limit(
346 FIELD => 'id',
347 OPERATOR => '>',
348 VALUE => $last,
349 ENTRYAGGREGATOR => 'none', # replaces last limit on this field
350 );
351 } else {
352 # XXX TODO: this could dig around inside the collection to see how it's
353 # limited and do the faster paging above under other conditions.
354 $self->SUPER::NextPage(@_);
355 }
356}
357
358sub Process {
359 my $self = shift;
360 my %args = (
361 object => undef,
362 @_
363 );
364
365 my $uid = $args{object}->UID;
366
367 # Skip all dependency walking if we're cloning. Marking UIDs as seen
368 # forces them to be visited immediately.
369 $self->{seen}{$uid}++
370 if $self->{Clone} and $uid;
371
372 return $self->SUPER::Process( @_ );
373}
374
375sub StackSize {
376 my $self = shift;
377 return scalar @{$self->{stack}};
378}
379
380sub ObjectCount {
381 my $self = shift;
382 return %{ $self->{ObjectCount} };
383}
384
385sub Observe {
386 my $self = shift;
387 my %args = (
388 object => undef,
389 direction => undef,
390 from => undef,
391 @_
392 );
393
394 my $obj = $args{object};
395 my $from = $args{from};
396 if ($obj->isa("RT::Ticket")) {
397 return 0 if $obj->Status eq "deleted" and not $self->{FollowDeleted};
398 return $self->{FollowTickets};
399 } elsif ($obj->isa("RT::ACE")) {
400 return $self->{FollowACL};
401 } elsif ($obj->isa("RT::Scrip") or $obj->isa("RT::Template")) {
402 return $self->{FollowScrips};
403 } elsif ($obj->isa("RT::GroupMember")) {
404 my $grp = $obj->GroupObj->Object;
405 if ($grp->Domain =~ /^RT::(Queue|Ticket)-Role$/) {
406 return 0 unless $grp->UID eq $from;
407 } elsif ($grp->Domain eq "SystemInternal") {
408 return 0 if $grp->UID eq $from;
409 }
410 }
411
412 return 1;
413}
414
415sub Visit {
416 my $self = shift;
417 my %args = (
418 object => undef,
419 @_
420 );
421
422 # Serialize it
423 my $obj = $args{object};
424 warn "Writing ".$obj->UID."\n" if $self->{Verbose};
425 my @store;
426 if ($obj->isa("RT::Migrate::Serializer::IncrementalRecord")) {
427 # These are stand-ins for record removals
428 my $class = $obj->ObjectType;
429 my %data = ( id => $obj->ObjectId );
430 # -class is used for transforms when dropping a record
431 if ($self->{Transform}{"-$class"}) {
432 $self->{Transform}{"-$class"}->(\%data,\$class)
433 }
434 @store = (
435 $class,
436 undef,
437 \%data,
438 );
439 } elsif ($self->{Clone}) {
440 # Short-circuit and get Just The Basics, Sir if we're cloning
441 my $class = ref($obj);
442 my $uid = $obj->UID;
443 my %data = $obj->RT::Record::Serialize( UIDs => 0 );
444
445 # +class is used when seeing a record of one class might insert
446 # a separate record into the stream
447 if ($self->{Transform}{"+$class"}) {
448 my @extra = $self->{Transform}{"+$class"}->(\%data,\$class);
449 for my $e (@extra) {
450 $! = 0;
451 Storable::nstore_fd($e, $self->{Filehandle});
452 die "Failed to write: $!" if $!;
453 $self->{ObjectCount}{$e->[0]}++;
454 }
455 }
456
457 # Upgrade the record if necessary
458 if ($self->{Transform}{$class}) {
459 $self->{Transform}{$class}->(\%data,\$class);
460 }
461
462 # Transforms set $class to undef to drop the record
463 return unless $class;
464
465 @store = (
466 $class,
467 $uid,
468 \%data,
469 );
470 } else {
471 @store = (
472 ref($obj),
473 $obj->UID,
474 { $obj->Serialize },
475 );
476 }
477
478 # Write it out; nstore_fd doesn't trap failures to write, so we have
479 # to; by clearing $! and checking it afterwards.
480 $! = 0;
481 Storable::nstore_fd(\@store, $self->{Filehandle});
482 die "Failed to write: $!" if $!;
483
484 $self->{ObjectCount}{$store[0]}++;
485}
486
4871;